1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the ARM target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_mod_imm(int64_t Imm) {
12
13 return ARM_AM::getSOImmVal(Arg: Imm) != -1;
14
15}
16static bool Predicate_imm0_7(int64_t Imm) {
17
18 return Imm >= 0 && Imm < 8;
19
20}
21static bool Predicate_imm0_255_expr(int64_t Imm) {
22 return Imm >= 0 && Imm < 256;
23}
24static bool Predicate_imm_sr(int64_t Imm) {
25
26 return Imm > 0 && Imm <= 32;
27
28}
29static bool Predicate_t2_so_imm(int64_t Imm) {
30
31 return ARM_AM::getT2SOImmVal(Arg: Imm) != -1;
32
33}
34static bool Predicate_imm0_4095(int64_t Imm) {
35
36 return Imm >= 0 && Imm < 4096;
37
38}
39static bool Predicate_imm1_31(int64_t Imm) {
40 return Imm > 0 && Imm < 32;
41}
42static bool Predicate_shr_imm8(int64_t Imm) {
43 return Imm > 0 && Imm <= 8;
44}
45static bool Predicate_shr_imm16(int64_t Imm) {
46 return Imm > 0 && Imm <= 16;
47}
48static bool Predicate_shr_imm32(int64_t Imm) {
49 return Imm > 0 && Imm <= 32;
50}
51static bool Predicate_VectorIndex32(int64_t Imm) {
52
53 return ((uint64_t)Imm) < 2;
54
55}
56static bool Predicate_imm0_31(int64_t Imm) {
57
58 return Imm >= 0 && Imm < 32;
59
60}
61static bool Predicate_mod_imm_not(int64_t Imm) {
62
63 return ARM_AM::getSOImmVal(Arg: ~(uint32_t)Imm) != -1;
64
65}
66static bool Predicate_imm0_65535(int64_t Imm) {
67
68 return Imm >= 0 && Imm < 65536;
69
70}
71static bool Predicate_t2_so_imm_neg(int64_t Imm) {
72
73 return Imm && ARM_AM::getT2SOImmVal(Arg: -(uint32_t)Imm) != -1;
74
75}
76static bool Predicate_imm0_15(int64_t Imm) {
77
78 return Imm >= 0 && Imm < 16;
79
80}
81
82
83// FastEmit functions for ISD::GET_FPENV.
84
85Register fastEmit_ISD_GET_FPENV_MVT_i32_(MVT RetVT) {
86 if (RetVT.SimpleTy != MVT::i32)
87 return Register();
88 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
89}
90
91Register fastEmit_ISD_GET_FPENV_(MVT VT, MVT RetVT) {
92 switch (VT.SimpleTy) {
93 case MVT::i32: return fastEmit_ISD_GET_FPENV_MVT_i32_(RetVT);
94 default: return Register();
95 }
96}
97
98// FastEmit functions for ISD::GET_FPMODE.
99
100Register fastEmit_ISD_GET_FPMODE_MVT_i32_(MVT RetVT) {
101 if (RetVT.SimpleTy != MVT::i32)
102 return Register();
103 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
104}
105
106Register fastEmit_ISD_GET_FPMODE_(MVT VT, MVT RetVT) {
107 switch (VT.SimpleTy) {
108 case MVT::i32: return fastEmit_ISD_GET_FPMODE_MVT_i32_(RetVT);
109 default: return Register();
110 }
111}
112
113// Top-level FastEmit function.
114
115Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
116 switch (Opcode) {
117 case ISD::GET_FPENV: return fastEmit_ISD_GET_FPENV_(VT, RetVT);
118 case ISD::GET_FPMODE: return fastEmit_ISD_GET_FPMODE_(VT, RetVT);
119 default: return Register();
120 }
121}
122
123// FastEmit functions for ARMISD::VADDVs.
124
125Register fastEmit_ARMISD_VADDVs_MVT_v16i8_r(MVT RetVT, Register Op0) {
126 if (RetVT.SimpleTy != MVT::i32)
127 return Register();
128 if ((Subtarget->hasMVEIntegerOps())) {
129 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
130 }
131 return Register();
132}
133
134Register fastEmit_ARMISD_VADDVs_MVT_v8i16_r(MVT RetVT, Register Op0) {
135 if (RetVT.SimpleTy != MVT::i32)
136 return Register();
137 if ((Subtarget->hasMVEIntegerOps())) {
138 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
139 }
140 return Register();
141}
142
143Register fastEmit_ARMISD_VADDVs_MVT_v4i32_r(MVT RetVT, Register Op0) {
144 if (RetVT.SimpleTy != MVT::i32)
145 return Register();
146 if ((Subtarget->hasMVEIntegerOps())) {
147 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
148 }
149 return Register();
150}
151
152Register fastEmit_ARMISD_VADDVs_r(MVT VT, MVT RetVT, Register Op0) {
153 switch (VT.SimpleTy) {
154 case MVT::v16i8: return fastEmit_ARMISD_VADDVs_MVT_v16i8_r(RetVT, Op0);
155 case MVT::v8i16: return fastEmit_ARMISD_VADDVs_MVT_v8i16_r(RetVT, Op0);
156 case MVT::v4i32: return fastEmit_ARMISD_VADDVs_MVT_v4i32_r(RetVT, Op0);
157 default: return Register();
158 }
159}
160
161// FastEmit functions for ARMISD::VADDVu.
162
163Register fastEmit_ARMISD_VADDVu_MVT_v16i8_r(MVT RetVT, Register Op0) {
164 if (RetVT.SimpleTy != MVT::i32)
165 return Register();
166 if ((Subtarget->hasMVEIntegerOps())) {
167 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
168 }
169 return Register();
170}
171
172Register fastEmit_ARMISD_VADDVu_MVT_v8i16_r(MVT RetVT, Register Op0) {
173 if (RetVT.SimpleTy != MVT::i32)
174 return Register();
175 if ((Subtarget->hasMVEIntegerOps())) {
176 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
177 }
178 return Register();
179}
180
181Register fastEmit_ARMISD_VADDVu_MVT_v4i32_r(MVT RetVT, Register Op0) {
182 if (RetVT.SimpleTy != MVT::i32)
183 return Register();
184 if ((Subtarget->hasMVEIntegerOps())) {
185 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
186 }
187 return Register();
188}
189
190Register fastEmit_ARMISD_VADDVu_r(MVT VT, MVT RetVT, Register Op0) {
191 switch (VT.SimpleTy) {
192 case MVT::v16i8: return fastEmit_ARMISD_VADDVu_MVT_v16i8_r(RetVT, Op0);
193 case MVT::v8i16: return fastEmit_ARMISD_VADDVu_MVT_v8i16_r(RetVT, Op0);
194 case MVT::v4i32: return fastEmit_ARMISD_VADDVu_MVT_v4i32_r(RetVT, Op0);
195 default: return Register();
196 }
197}
198
199// FastEmit functions for ARMISD::VDUP.
200
201Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Register Op0) {
202 if ((Subtarget->hasNEON())) {
203 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8d, RC: &ARM::DPRRegClass, Op0);
204 }
205 return Register();
206}
207
208Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Register Op0) {
209 if ((Subtarget->hasMVEIntegerOps())) {
210 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP8, RC: &ARM::MQPRRegClass, Op0);
211 }
212 if ((Subtarget->hasNEON())) {
213 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8q, RC: &ARM::QPRRegClass, Op0);
214 }
215 return Register();
216}
217
218Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Register Op0) {
219 if ((Subtarget->hasNEON())) {
220 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16d, RC: &ARM::DPRRegClass, Op0);
221 }
222 return Register();
223}
224
225Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Register Op0) {
226 if ((Subtarget->hasMVEIntegerOps())) {
227 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
228 }
229 if ((Subtarget->hasNEON())) {
230 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16q, RC: &ARM::QPRRegClass, Op0);
231 }
232 return Register();
233}
234
235Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Register Op0) {
236 if ((!Subtarget->hasSlowVDUP32()) && (Subtarget->hasNEON())) {
237 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32d, RC: &ARM::DPRRegClass, Op0);
238 }
239 return Register();
240}
241
242Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Register Op0) {
243 if ((Subtarget->hasMVEIntegerOps())) {
244 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
245 }
246 if ((Subtarget->hasNEON())) {
247 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32q, RC: &ARM::QPRRegClass, Op0);
248 }
249 return Register();
250}
251
252Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Register Op0) {
253 if ((Subtarget->hasMVEIntegerOps())) {
254 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
255 }
256 return Register();
257}
258
259Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Register Op0) {
260 if ((Subtarget->hasMVEIntegerOps())) {
261 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
262 }
263 return Register();
264}
265
266Register fastEmit_ARMISD_VDUP_MVT_i32_r(MVT RetVT, Register Op0) {
267switch (RetVT.SimpleTy) {
268 case MVT::v8i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Op0);
269 case MVT::v16i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Op0);
270 case MVT::v4i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Op0);
271 case MVT::v8i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Op0);
272 case MVT::v2i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Op0);
273 case MVT::v4i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Op0);
274 case MVT::v8f16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Op0);
275 case MVT::v4f32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Op0);
276 default: return Register();
277}
278}
279
280Register fastEmit_ARMISD_VDUP_r(MVT VT, MVT RetVT, Register Op0) {
281 switch (VT.SimpleTy) {
282 case MVT::i32: return fastEmit_ARMISD_VDUP_MVT_i32_r(RetVT, Op0);
283 default: return Register();
284 }
285}
286
287// FastEmit functions for ARMISD::VMOVSR.
288
289Register fastEmit_ARMISD_VMOVSR_MVT_i32_r(MVT RetVT, Register Op0) {
290 if (RetVT.SimpleTy != MVT::f32)
291 return Register();
292 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
293 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
294 }
295 return Register();
296}
297
298Register fastEmit_ARMISD_VMOVSR_r(MVT VT, MVT RetVT, Register Op0) {
299 switch (VT.SimpleTy) {
300 case MVT::i32: return fastEmit_ARMISD_VMOVSR_MVT_i32_r(RetVT, Op0);
301 default: return Register();
302 }
303}
304
305// FastEmit functions for ARMISD::VMOVhr.
306
307Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Register Op0) {
308 if ((Subtarget->hasFPRegs16())) {
309 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
310 }
311 return Register();
312}
313
314Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Register Op0) {
315 if ((Subtarget->hasFPRegs16())) {
316 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
317 }
318 return Register();
319}
320
321Register fastEmit_ARMISD_VMOVhr_MVT_i32_r(MVT RetVT, Register Op0) {
322switch (RetVT.SimpleTy) {
323 case MVT::bf16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Op0);
324 case MVT::f16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Op0);
325 default: return Register();
326}
327}
328
329Register fastEmit_ARMISD_VMOVhr_r(MVT VT, MVT RetVT, Register Op0) {
330 switch (VT.SimpleTy) {
331 case MVT::i32: return fastEmit_ARMISD_VMOVhr_MVT_i32_r(RetVT, Op0);
332 default: return Register();
333 }
334}
335
336// FastEmit functions for ARMISD::VMOVrh.
337
338Register fastEmit_ARMISD_VMOVrh_MVT_bf16_r(MVT RetVT, Register Op0) {
339 if (RetVT.SimpleTy != MVT::i32)
340 return Register();
341 if ((Subtarget->hasFPRegs16())) {
342 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
343 }
344 return Register();
345}
346
347Register fastEmit_ARMISD_VMOVrh_MVT_f16_r(MVT RetVT, Register Op0) {
348 if (RetVT.SimpleTy != MVT::i32)
349 return Register();
350 if ((Subtarget->hasFPRegs16())) {
351 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
352 }
353 return Register();
354}
355
356Register fastEmit_ARMISD_VMOVrh_r(MVT VT, MVT RetVT, Register Op0) {
357 switch (VT.SimpleTy) {
358 case MVT::bf16: return fastEmit_ARMISD_VMOVrh_MVT_bf16_r(RetVT, Op0);
359 case MVT::f16: return fastEmit_ARMISD_VMOVrh_MVT_f16_r(RetVT, Op0);
360 default: return Register();
361 }
362}
363
364// FastEmit functions for ARMISD::VREV16.
365
366Register fastEmit_ARMISD_VREV16_MVT_v8i8_r(MVT RetVT, Register Op0) {
367 if (RetVT.SimpleTy != MVT::v8i8)
368 return Register();
369 if ((Subtarget->hasNEON())) {
370 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
371 }
372 return Register();
373}
374
375Register fastEmit_ARMISD_VREV16_MVT_v16i8_r(MVT RetVT, Register Op0) {
376 if (RetVT.SimpleTy != MVT::v16i8)
377 return Register();
378 if ((Subtarget->hasMVEIntegerOps())) {
379 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
380 }
381 if ((Subtarget->hasNEON())) {
382 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
383 }
384 return Register();
385}
386
387Register fastEmit_ARMISD_VREV16_r(MVT VT, MVT RetVT, Register Op0) {
388 switch (VT.SimpleTy) {
389 case MVT::v8i8: return fastEmit_ARMISD_VREV16_MVT_v8i8_r(RetVT, Op0);
390 case MVT::v16i8: return fastEmit_ARMISD_VREV16_MVT_v16i8_r(RetVT, Op0);
391 default: return Register();
392 }
393}
394
395// FastEmit functions for ARMISD::VREV32.
396
397Register fastEmit_ARMISD_VREV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
398 if (RetVT.SimpleTy != MVT::v8i8)
399 return Register();
400 if ((Subtarget->hasNEON())) {
401 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
402 }
403 return Register();
404}
405
406Register fastEmit_ARMISD_VREV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
407 if (RetVT.SimpleTy != MVT::v16i8)
408 return Register();
409 if ((Subtarget->hasMVEIntegerOps())) {
410 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
411 }
412 if ((Subtarget->hasNEON())) {
413 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
414 }
415 return Register();
416}
417
418Register fastEmit_ARMISD_VREV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
419 if (RetVT.SimpleTy != MVT::v4i16)
420 return Register();
421 if ((Subtarget->hasNEON())) {
422 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
423 }
424 return Register();
425}
426
427Register fastEmit_ARMISD_VREV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
428 if (RetVT.SimpleTy != MVT::v8i16)
429 return Register();
430 if ((Subtarget->hasMVEIntegerOps())) {
431 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
432 }
433 if ((Subtarget->hasNEON())) {
434 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
435 }
436 return Register();
437}
438
439Register fastEmit_ARMISD_VREV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
440 if (RetVT.SimpleTy != MVT::v4f16)
441 return Register();
442 if ((Subtarget->hasNEON())) {
443 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
444 }
445 return Register();
446}
447
448Register fastEmit_ARMISD_VREV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
449 if (RetVT.SimpleTy != MVT::v8f16)
450 return Register();
451 if ((Subtarget->hasMVEIntegerOps())) {
452 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
453 }
454 if ((Subtarget->hasNEON())) {
455 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
456 }
457 return Register();
458}
459
460Register fastEmit_ARMISD_VREV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
461 if (RetVT.SimpleTy != MVT::v4bf16)
462 return Register();
463 if ((Subtarget->hasNEON())) {
464 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
465 }
466 return Register();
467}
468
469Register fastEmit_ARMISD_VREV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
470 if (RetVT.SimpleTy != MVT::v8bf16)
471 return Register();
472 if ((Subtarget->hasNEON())) {
473 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
474 }
475 return Register();
476}
477
478Register fastEmit_ARMISD_VREV32_r(MVT VT, MVT RetVT, Register Op0) {
479 switch (VT.SimpleTy) {
480 case MVT::v8i8: return fastEmit_ARMISD_VREV32_MVT_v8i8_r(RetVT, Op0);
481 case MVT::v16i8: return fastEmit_ARMISD_VREV32_MVT_v16i8_r(RetVT, Op0);
482 case MVT::v4i16: return fastEmit_ARMISD_VREV32_MVT_v4i16_r(RetVT, Op0);
483 case MVT::v8i16: return fastEmit_ARMISD_VREV32_MVT_v8i16_r(RetVT, Op0);
484 case MVT::v4f16: return fastEmit_ARMISD_VREV32_MVT_v4f16_r(RetVT, Op0);
485 case MVT::v8f16: return fastEmit_ARMISD_VREV32_MVT_v8f16_r(RetVT, Op0);
486 case MVT::v4bf16: return fastEmit_ARMISD_VREV32_MVT_v4bf16_r(RetVT, Op0);
487 case MVT::v8bf16: return fastEmit_ARMISD_VREV32_MVT_v8bf16_r(RetVT, Op0);
488 default: return Register();
489 }
490}
491
492// FastEmit functions for ARMISD::VREV64.
493
494Register fastEmit_ARMISD_VREV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
495 if (RetVT.SimpleTy != MVT::v8i8)
496 return Register();
497 if ((Subtarget->hasNEON())) {
498 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
499 }
500 return Register();
501}
502
503Register fastEmit_ARMISD_VREV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
504 if (RetVT.SimpleTy != MVT::v16i8)
505 return Register();
506 if ((Subtarget->hasMVEIntegerOps())) {
507 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
508 }
509 if ((Subtarget->hasNEON())) {
510 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
511 }
512 return Register();
513}
514
515Register fastEmit_ARMISD_VREV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
516 if (RetVT.SimpleTy != MVT::v4i16)
517 return Register();
518 if ((Subtarget->hasNEON())) {
519 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
520 }
521 return Register();
522}
523
524Register fastEmit_ARMISD_VREV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
525 if (RetVT.SimpleTy != MVT::v8i16)
526 return Register();
527 if ((Subtarget->hasMVEIntegerOps())) {
528 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
529 }
530 if ((Subtarget->hasNEON())) {
531 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
532 }
533 return Register();
534}
535
536Register fastEmit_ARMISD_VREV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
537 if (RetVT.SimpleTy != MVT::v2i32)
538 return Register();
539 if ((Subtarget->hasNEON())) {
540 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
541 }
542 return Register();
543}
544
545Register fastEmit_ARMISD_VREV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
546 if (RetVT.SimpleTy != MVT::v4i32)
547 return Register();
548 if ((Subtarget->hasMVEIntegerOps())) {
549 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
550 }
551 if ((Subtarget->hasNEON())) {
552 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
553 }
554 return Register();
555}
556
557Register fastEmit_ARMISD_VREV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
558 if (RetVT.SimpleTy != MVT::v4f16)
559 return Register();
560 if ((Subtarget->hasNEON())) {
561 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
562 }
563 return Register();
564}
565
566Register fastEmit_ARMISD_VREV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
567 if (RetVT.SimpleTy != MVT::v8f16)
568 return Register();
569 if ((Subtarget->hasMVEIntegerOps())) {
570 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
571 }
572 if ((Subtarget->hasNEON())) {
573 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
574 }
575 return Register();
576}
577
578Register fastEmit_ARMISD_VREV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
579 if (RetVT.SimpleTy != MVT::v4bf16)
580 return Register();
581 if ((Subtarget->hasNEON())) {
582 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
583 }
584 return Register();
585}
586
587Register fastEmit_ARMISD_VREV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
588 if (RetVT.SimpleTy != MVT::v8bf16)
589 return Register();
590 if ((Subtarget->hasNEON())) {
591 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
592 }
593 return Register();
594}
595
596Register fastEmit_ARMISD_VREV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
597 if (RetVT.SimpleTy != MVT::v2f32)
598 return Register();
599 if ((Subtarget->hasNEON())) {
600 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
601 }
602 return Register();
603}
604
605Register fastEmit_ARMISD_VREV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
606 if (RetVT.SimpleTy != MVT::v4f32)
607 return Register();
608 if ((Subtarget->hasMVEIntegerOps())) {
609 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
610 }
611 if ((Subtarget->hasNEON())) {
612 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
613 }
614 return Register();
615}
616
617Register fastEmit_ARMISD_VREV64_r(MVT VT, MVT RetVT, Register Op0) {
618 switch (VT.SimpleTy) {
619 case MVT::v8i8: return fastEmit_ARMISD_VREV64_MVT_v8i8_r(RetVT, Op0);
620 case MVT::v16i8: return fastEmit_ARMISD_VREV64_MVT_v16i8_r(RetVT, Op0);
621 case MVT::v4i16: return fastEmit_ARMISD_VREV64_MVT_v4i16_r(RetVT, Op0);
622 case MVT::v8i16: return fastEmit_ARMISD_VREV64_MVT_v8i16_r(RetVT, Op0);
623 case MVT::v2i32: return fastEmit_ARMISD_VREV64_MVT_v2i32_r(RetVT, Op0);
624 case MVT::v4i32: return fastEmit_ARMISD_VREV64_MVT_v4i32_r(RetVT, Op0);
625 case MVT::v4f16: return fastEmit_ARMISD_VREV64_MVT_v4f16_r(RetVT, Op0);
626 case MVT::v8f16: return fastEmit_ARMISD_VREV64_MVT_v8f16_r(RetVT, Op0);
627 case MVT::v4bf16: return fastEmit_ARMISD_VREV64_MVT_v4bf16_r(RetVT, Op0);
628 case MVT::v8bf16: return fastEmit_ARMISD_VREV64_MVT_v8bf16_r(RetVT, Op0);
629 case MVT::v2f32: return fastEmit_ARMISD_VREV64_MVT_v2f32_r(RetVT, Op0);
630 case MVT::v4f32: return fastEmit_ARMISD_VREV64_MVT_v4f32_r(RetVT, Op0);
631 default: return Register();
632 }
633}
634
635// FastEmit functions for ARMISD::WLSSETUP.
636
637Register fastEmit_ARMISD_WLSSETUP_MVT_i32_r(MVT RetVT, Register Op0) {
638 if (RetVT.SimpleTy != MVT::i32)
639 return Register();
640 if ((Subtarget->hasLOB()) && (Subtarget->hasV8_1MMainlineOps()) && (Subtarget->isThumb2())) {
641 return fastEmitInst_r(MachineInstOpcode: ARM::t2WhileLoopSetup, RC: &ARM::GPRlrRegClass, Op0);
642 }
643 return Register();
644}
645
646Register fastEmit_ARMISD_WLSSETUP_r(MVT VT, MVT RetVT, Register Op0) {
647 switch (VT.SimpleTy) {
648 case MVT::i32: return fastEmit_ARMISD_WLSSETUP_MVT_i32_r(RetVT, Op0);
649 default: return Register();
650 }
651}
652
653// FastEmit functions for ISD::ABS.
654
655Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
656 if (RetVT.SimpleTy != MVT::v8i8)
657 return Register();
658 if ((Subtarget->hasNEON())) {
659 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i8, RC: &ARM::DPRRegClass, Op0);
660 }
661 return Register();
662}
663
664Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
665 if (RetVT.SimpleTy != MVT::v16i8)
666 return Register();
667 if ((Subtarget->hasMVEIntegerOps())) {
668 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs8, RC: &ARM::MQPRRegClass, Op0);
669 }
670 if ((Subtarget->hasNEON())) {
671 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv16i8, RC: &ARM::QPRRegClass, Op0);
672 }
673 return Register();
674}
675
676Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
677 if (RetVT.SimpleTy != MVT::v4i16)
678 return Register();
679 if ((Subtarget->hasNEON())) {
680 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i16, RC: &ARM::DPRRegClass, Op0);
681 }
682 return Register();
683}
684
685Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
686 if (RetVT.SimpleTy != MVT::v8i16)
687 return Register();
688 if ((Subtarget->hasMVEIntegerOps())) {
689 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs16, RC: &ARM::MQPRRegClass, Op0);
690 }
691 if ((Subtarget->hasNEON())) {
692 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i16, RC: &ARM::QPRRegClass, Op0);
693 }
694 return Register();
695}
696
697Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
698 if (RetVT.SimpleTy != MVT::v2i32)
699 return Register();
700 if ((Subtarget->hasNEON())) {
701 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv2i32, RC: &ARM::DPRRegClass, Op0);
702 }
703 return Register();
704}
705
706Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
707 if (RetVT.SimpleTy != MVT::v4i32)
708 return Register();
709 if ((Subtarget->hasMVEIntegerOps())) {
710 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs32, RC: &ARM::MQPRRegClass, Op0);
711 }
712 if ((Subtarget->hasNEON())) {
713 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i32, RC: &ARM::QPRRegClass, Op0);
714 }
715 return Register();
716}
717
718Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
719 switch (VT.SimpleTy) {
720 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
721 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
722 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
723 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
724 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
725 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
726 default: return Register();
727 }
728}
729
730// FastEmit functions for ISD::ANY_EXTEND.
731
732Register fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
733 if (RetVT.SimpleTy != MVT::v8i16)
734 return Register();
735 if ((Subtarget->hasNEON())) {
736 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
737 }
738 return Register();
739}
740
741Register fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
742 if (RetVT.SimpleTy != MVT::v4i32)
743 return Register();
744 if ((Subtarget->hasNEON())) {
745 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
746 }
747 return Register();
748}
749
750Register fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
751 if (RetVT.SimpleTy != MVT::v2i64)
752 return Register();
753 if ((Subtarget->hasNEON())) {
754 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
755 }
756 return Register();
757}
758
759Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
760 switch (VT.SimpleTy) {
761 case MVT::v8i8: return fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(RetVT, Op0);
762 case MVT::v4i16: return fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(RetVT, Op0);
763 case MVT::v2i32: return fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(RetVT, Op0);
764 default: return Register();
765 }
766}
767
768// FastEmit functions for ISD::BITCAST.
769
770Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
771 if (RetVT.SimpleTy != MVT::f32)
772 return Register();
773 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
774 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
775 }
776 return Register();
777}
778
779Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
780 if (RetVT.SimpleTy != MVT::i32)
781 return Register();
782 if ((Subtarget->hasFPRegs())) {
783 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRS, RC: &ARM::GPRRegClass, Op0);
784 }
785 return Register();
786}
787
788Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
789 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
790 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
791 }
792 return Register();
793}
794
795Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
796 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
797 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
798 }
799 return Register();
800}
801
802Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
803 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
804 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
805 }
806 return Register();
807}
808
809Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
810 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
811 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
812 }
813 return Register();
814}
815
816Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
817 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
818 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
819 }
820 return Register();
821}
822
823Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
824 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
825 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
826 }
827 return Register();
828}
829
830Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
831switch (RetVT.SimpleTy) {
832 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
833 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
834 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
835 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
836 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
837 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
838 default: return Register();
839}
840}
841
842Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
843 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
844 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
845 }
846 return Register();
847}
848
849Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
850 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
851 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
852 }
853 return Register();
854}
855
856Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
857 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
858 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
859 }
860 return Register();
861}
862
863Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
864 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
865 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
866 }
867 return Register();
868}
869
870Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
871 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
872 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
873 }
874 return Register();
875}
876
877Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
878 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
879 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
880 }
881 return Register();
882}
883
884Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
885 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
886 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
887 }
888 return Register();
889}
890
891Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
892switch (RetVT.SimpleTy) {
893 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
894 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
895 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
896 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
897 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
898 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
899 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
900 default: return Register();
901}
902}
903
904Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
905 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
906 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
907 }
908 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
909 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
910 }
911 return Register();
912}
913
914Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
915 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
916 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
917 }
918 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
919 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
920 }
921 return Register();
922}
923
924Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
925 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
926 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
927 }
928 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
929 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
930 }
931 return Register();
932}
933
934Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
935 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
936 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
937 }
938 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
939 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
940 }
941 return Register();
942}
943
944Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
945 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
946 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
947 }
948 return Register();
949}
950
951Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
952 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
953 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
954 }
955 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
956 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
957 }
958 return Register();
959}
960
961Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
962 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
963 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
964 }
965 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
966 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
967 }
968 return Register();
969}
970
971Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
972switch (RetVT.SimpleTy) {
973 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
974 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
975 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
976 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
977 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
978 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
979 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
980 default: return Register();
981}
982}
983
984Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
985 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
986 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
987 }
988 return Register();
989}
990
991Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
992 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
993 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
994 }
995 return Register();
996}
997
998Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
999 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1000 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1001 }
1002 return Register();
1003}
1004
1005Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1006 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1007 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1008 }
1009 return Register();
1010}
1011
1012Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1013 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1014 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1015 }
1016 return Register();
1017}
1018
1019Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1020switch (RetVT.SimpleTy) {
1021 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1022 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1023 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1024 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1025 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1026 default: return Register();
1027}
1028}
1029
1030Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1031 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1032 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1033 }
1034 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1035 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1036 }
1037 return Register();
1038}
1039
1040Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1041 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1042 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1043 }
1044 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1045 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1046 }
1047 return Register();
1048}
1049
1050Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1051 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1052 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1053 }
1054 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1055 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1056 }
1057 return Register();
1058}
1059
1060Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1061 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1062 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1063 }
1064 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1065 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1066 }
1067 return Register();
1068}
1069
1070Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1071 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1072 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1073 }
1074 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1075 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1076 }
1077 return Register();
1078}
1079
1080Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1081switch (RetVT.SimpleTy) {
1082 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1083 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1084 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1085 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1086 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1087 default: return Register();
1088}
1089}
1090
1091Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1092 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1093 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1094 }
1095 return Register();
1096}
1097
1098Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1099 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1100 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1101 }
1102 return Register();
1103}
1104
1105Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1106 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1107 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1108 }
1109 return Register();
1110}
1111
1112Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1113 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1114 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1115 }
1116 return Register();
1117}
1118
1119Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1120 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1121 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1122 }
1123 return Register();
1124}
1125
1126Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1127 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1128 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1129 }
1130 return Register();
1131}
1132
1133Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1134switch (RetVT.SimpleTy) {
1135 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1136 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1137 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1138 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1139 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1140 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1141 default: return Register();
1142}
1143}
1144
1145Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1146 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1147 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1148 }
1149 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1150 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1151 }
1152 return Register();
1153}
1154
1155Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1156 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1157 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1158 }
1159 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1160 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1161 }
1162 return Register();
1163}
1164
1165Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1166 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1167 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1168 }
1169 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1170 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1171 }
1172 return Register();
1173}
1174
1175Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1176 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1177 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1178 }
1179 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1180 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1181 }
1182 return Register();
1183}
1184
1185Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1186 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1187 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1188 }
1189 return Register();
1190}
1191
1192Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1193 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1194 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1195 }
1196 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1197 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1198 }
1199 return Register();
1200}
1201
1202Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1203switch (RetVT.SimpleTy) {
1204 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1205 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1206 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1207 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1208 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1209 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1210 default: return Register();
1211}
1212}
1213
1214Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1215 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1216 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1217 }
1218 return Register();
1219}
1220
1221Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1222 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1223 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1224 }
1225 return Register();
1226}
1227
1228Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1229 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1230 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1231 }
1232 return Register();
1233}
1234
1235Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1236 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1237 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1238 }
1239 return Register();
1240}
1241
1242Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1243 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1244 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1245 }
1246 return Register();
1247}
1248
1249Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1250 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1251 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1252 }
1253 return Register();
1254}
1255
1256Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1257switch (RetVT.SimpleTy) {
1258 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1259 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1260 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1261 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1262 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1263 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1264 default: return Register();
1265}
1266}
1267
1268Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1269 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1270 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1271 }
1272 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1273 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1274 }
1275 return Register();
1276}
1277
1278Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1279 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1280 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1281 }
1282 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1283 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1284 }
1285 return Register();
1286}
1287
1288Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1289 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1290 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1291 }
1292 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1293 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1294 }
1295 return Register();
1296}
1297
1298Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1299 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1300 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1301 }
1302 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1303 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1304 }
1305 return Register();
1306}
1307
1308Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1309 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1310 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1311 }
1312 return Register();
1313}
1314
1315Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1316 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1317 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1318 }
1319 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1320 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1321 }
1322 return Register();
1323}
1324
1325Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1326switch (RetVT.SimpleTy) {
1327 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1328 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1329 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1330 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1331 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1332 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1333 default: return Register();
1334}
1335}
1336
1337Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1338 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1339 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1340 }
1341 return Register();
1342}
1343
1344Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1345 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1346 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1347 }
1348 return Register();
1349}
1350
1351Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1352 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1353 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1354 }
1355 return Register();
1356}
1357
1358Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1359 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1360 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1361 }
1362 return Register();
1363}
1364
1365Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1366 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1367 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1368 }
1369 return Register();
1370}
1371
1372Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1373switch (RetVT.SimpleTy) {
1374 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1375 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1376 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1377 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1378 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1379 default: return Register();
1380}
1381}
1382
1383Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1384 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1385 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1386 }
1387 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1388 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1389 }
1390 return Register();
1391}
1392
1393Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1394 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1395 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1396 }
1397 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1398 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1399 }
1400 return Register();
1401}
1402
1403Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1404 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1405 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1406 }
1407 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1408 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1409 }
1410 return Register();
1411}
1412
1413Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1414 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1415 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1416 }
1417 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1418 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1419 }
1420 return Register();
1421}
1422
1423Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1424 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1425 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1426 }
1427 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1428 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1429 }
1430 return Register();
1431}
1432
1433Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1434switch (RetVT.SimpleTy) {
1435 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1436 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1437 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1438 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1439 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1440 default: return Register();
1441}
1442}
1443
1444Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1445 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1446 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1447 }
1448 return Register();
1449}
1450
1451Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1452 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1453 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1454 }
1455 return Register();
1456}
1457
1458Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1459 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1460 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1461 }
1462 return Register();
1463}
1464
1465Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1466 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1467 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1468 }
1469 return Register();
1470}
1471
1472Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1473 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1474 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1475 }
1476 return Register();
1477}
1478
1479Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1480switch (RetVT.SimpleTy) {
1481 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1482 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1483 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1484 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1485 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1486 default: return Register();
1487}
1488}
1489
1490Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1491 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1492 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1493 }
1494 return Register();
1495}
1496
1497Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1498 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1499 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1500 }
1501 return Register();
1502}
1503
1504Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1505 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1506 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1507 }
1508 return Register();
1509}
1510
1511Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1512 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1513 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1514 }
1515 return Register();
1516}
1517
1518Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1519 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1520 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1521 }
1522 return Register();
1523}
1524
1525Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1526switch (RetVT.SimpleTy) {
1527 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1528 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1529 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1530 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1531 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1532 default: return Register();
1533}
1534}
1535
1536Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1537 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1538 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1539 }
1540 return Register();
1541}
1542
1543Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1544 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1545 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1546 }
1547 return Register();
1548}
1549
1550Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1551 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1552 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1553 }
1554 return Register();
1555}
1556
1557Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
1558 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1559 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1560 }
1561 return Register();
1562}
1563
1564Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
1565 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1566 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1567 }
1568 return Register();
1569}
1570
1571Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
1572 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1573 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1574 }
1575 return Register();
1576}
1577
1578Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
1579switch (RetVT.SimpleTy) {
1580 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
1581 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
1582 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
1583 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
1584 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
1585 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
1586 default: return Register();
1587}
1588}
1589
1590Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
1591 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1592 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1593 }
1594 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1595 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1596 }
1597 return Register();
1598}
1599
1600Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
1601 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1602 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1603 }
1604 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1605 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1606 }
1607 return Register();
1608}
1609
1610Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
1611 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1612 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1613 }
1614 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1615 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1616 }
1617 return Register();
1618}
1619
1620Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
1621 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1622 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1623 }
1624 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1625 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1626 }
1627 return Register();
1628}
1629
1630Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
1631 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1632 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1633 }
1634 return Register();
1635}
1636
1637Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
1638 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1639 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1640 }
1641 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1642 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1643 }
1644 return Register();
1645}
1646
1647Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
1648switch (RetVT.SimpleTy) {
1649 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
1650 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
1651 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
1652 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
1653 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
1654 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
1655 default: return Register();
1656}
1657}
1658
1659Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
1660 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1661 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1662 }
1663 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1664 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1665 }
1666 return Register();
1667}
1668
1669Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
1670 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1671 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1672 }
1673 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1674 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1675 }
1676 return Register();
1677}
1678
1679Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
1680 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1681 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1682 }
1683 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1684 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1685 }
1686 return Register();
1687}
1688
1689Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
1690 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1691 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1692 }
1693 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1694 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1695 }
1696 return Register();
1697}
1698
1699Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
1700 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1701 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1702 }
1703 return Register();
1704}
1705
1706Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
1707 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1708 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1709 }
1710 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1711 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1712 }
1713 return Register();
1714}
1715
1716Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
1717switch (RetVT.SimpleTy) {
1718 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
1719 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
1720 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
1721 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
1722 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
1723 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
1724 default: return Register();
1725}
1726}
1727
1728Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
1729 switch (VT.SimpleTy) {
1730 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
1731 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
1732 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
1733 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
1734 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
1735 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
1736 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
1737 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
1738 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
1739 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
1740 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
1741 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
1742 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
1743 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
1744 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
1745 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
1746 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
1747 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
1748 default: return Register();
1749 }
1750}
1751
1752// FastEmit functions for ISD::BITREVERSE.
1753
1754Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
1755 if (RetVT.SimpleTy != MVT::i32)
1756 return Register();
1757 if ((Subtarget->isThumb2())) {
1758 return fastEmitInst_r(MachineInstOpcode: ARM::t2RBIT, RC: &ARM::rGPRRegClass, Op0);
1759 }
1760 if ((Subtarget->hasV6T2Ops()) && (!Subtarget->isThumb())) {
1761 return fastEmitInst_r(MachineInstOpcode: ARM::RBIT, RC: &ARM::GPRRegClass, Op0);
1762 }
1763 return Register();
1764}
1765
1766Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
1767 switch (VT.SimpleTy) {
1768 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
1769 default: return Register();
1770 }
1771}
1772
1773// FastEmit functions for ISD::BSWAP.
1774
1775Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
1776 if (RetVT.SimpleTy != MVT::i32)
1777 return Register();
1778 if ((Subtarget->isThumb2())) {
1779 return fastEmitInst_r(MachineInstOpcode: ARM::t2REV, RC: &ARM::rGPRRegClass, Op0);
1780 }
1781 if ((Subtarget->hasV6Ops()) && (Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
1782 return fastEmitInst_r(MachineInstOpcode: ARM::tREV, RC: &ARM::tGPRRegClass, Op0);
1783 }
1784 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
1785 return fastEmitInst_r(MachineInstOpcode: ARM::REV, RC: &ARM::GPRRegClass, Op0);
1786 }
1787 return Register();
1788}
1789
1790Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1791 if (RetVT.SimpleTy != MVT::v8i16)
1792 return Register();
1793 if ((Subtarget->hasMVEIntegerOps())) {
1794 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1795 }
1796 return Register();
1797}
1798
1799Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1800 if (RetVT.SimpleTy != MVT::v4i32)
1801 return Register();
1802 if ((Subtarget->hasMVEIntegerOps())) {
1803 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1804 }
1805 return Register();
1806}
1807
1808Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
1809 switch (VT.SimpleTy) {
1810 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
1811 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
1812 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
1813 default: return Register();
1814 }
1815}
1816
1817// FastEmit functions for ISD::CTLS.
1818
1819Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1820 if (RetVT.SimpleTy != MVT::v8i8)
1821 return Register();
1822 if ((Subtarget->hasNEON())) {
1823 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv8i8, RC: &ARM::DPRRegClass, Op0);
1824 }
1825 return Register();
1826}
1827
1828Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1829 if (RetVT.SimpleTy != MVT::v16i8)
1830 return Register();
1831 if ((Subtarget->hasMVEIntegerOps())) {
1832 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs8, RC: &ARM::MQPRRegClass, Op0);
1833 }
1834 if ((Subtarget->hasNEON())) {
1835 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv16i8, RC: &ARM::QPRRegClass, Op0);
1836 }
1837 return Register();
1838}
1839
1840Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1841 if (RetVT.SimpleTy != MVT::v4i16)
1842 return Register();
1843 if ((Subtarget->hasNEON())) {
1844 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv4i16, RC: &ARM::DPRRegClass, Op0);
1845 }
1846 return Register();
1847}
1848
1849Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1850 if (RetVT.SimpleTy != MVT::v8i16)
1851 return Register();
1852 if ((Subtarget->hasMVEIntegerOps())) {
1853 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs16, RC: &ARM::MQPRRegClass, Op0);
1854 }
1855 if ((Subtarget->hasNEON())) {
1856 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv8i16, RC: &ARM::QPRRegClass, Op0);
1857 }
1858 return Register();
1859}
1860
1861Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1862 if (RetVT.SimpleTy != MVT::v2i32)
1863 return Register();
1864 if ((Subtarget->hasNEON())) {
1865 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv2i32, RC: &ARM::DPRRegClass, Op0);
1866 }
1867 return Register();
1868}
1869
1870Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1871 if (RetVT.SimpleTy != MVT::v4i32)
1872 return Register();
1873 if ((Subtarget->hasMVEIntegerOps())) {
1874 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs32, RC: &ARM::MQPRRegClass, Op0);
1875 }
1876 if ((Subtarget->hasNEON())) {
1877 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv4i32, RC: &ARM::QPRRegClass, Op0);
1878 }
1879 return Register();
1880}
1881
1882Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
1883 switch (VT.SimpleTy) {
1884 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
1885 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
1886 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
1887 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
1888 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
1889 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
1890 default: return Register();
1891 }
1892}
1893
1894// FastEmit functions for ISD::CTLZ.
1895
1896Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
1897 if (RetVT.SimpleTy != MVT::i32)
1898 return Register();
1899 if ((Subtarget->isThumb2())) {
1900 return fastEmitInst_r(MachineInstOpcode: ARM::t2CLZ, RC: &ARM::rGPRRegClass, Op0);
1901 }
1902 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb())) {
1903 return fastEmitInst_r(MachineInstOpcode: ARM::CLZ, RC: &ARM::GPRRegClass, Op0);
1904 }
1905 return Register();
1906}
1907
1908Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
1909 if (RetVT.SimpleTy != MVT::v8i8)
1910 return Register();
1911 if ((Subtarget->hasNEON())) {
1912 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i8, RC: &ARM::DPRRegClass, Op0);
1913 }
1914 return Register();
1915}
1916
1917Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
1918 if (RetVT.SimpleTy != MVT::v16i8)
1919 return Register();
1920 if ((Subtarget->hasMVEIntegerOps())) {
1921 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs8, RC: &ARM::MQPRRegClass, Op0);
1922 }
1923 if ((Subtarget->hasNEON())) {
1924 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv16i8, RC: &ARM::QPRRegClass, Op0);
1925 }
1926 return Register();
1927}
1928
1929Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
1930 if (RetVT.SimpleTy != MVT::v4i16)
1931 return Register();
1932 if ((Subtarget->hasNEON())) {
1933 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i16, RC: &ARM::DPRRegClass, Op0);
1934 }
1935 return Register();
1936}
1937
1938Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
1939 if (RetVT.SimpleTy != MVT::v8i16)
1940 return Register();
1941 if ((Subtarget->hasMVEIntegerOps())) {
1942 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs16, RC: &ARM::MQPRRegClass, Op0);
1943 }
1944 if ((Subtarget->hasNEON())) {
1945 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i16, RC: &ARM::QPRRegClass, Op0);
1946 }
1947 return Register();
1948}
1949
1950Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
1951 if (RetVT.SimpleTy != MVT::v2i32)
1952 return Register();
1953 if ((Subtarget->hasNEON())) {
1954 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv2i32, RC: &ARM::DPRRegClass, Op0);
1955 }
1956 return Register();
1957}
1958
1959Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
1960 if (RetVT.SimpleTy != MVT::v4i32)
1961 return Register();
1962 if ((Subtarget->hasMVEIntegerOps())) {
1963 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs32, RC: &ARM::MQPRRegClass, Op0);
1964 }
1965 if ((Subtarget->hasNEON())) {
1966 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i32, RC: &ARM::QPRRegClass, Op0);
1967 }
1968 return Register();
1969}
1970
1971Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
1972 switch (VT.SimpleTy) {
1973 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
1974 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
1975 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
1976 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
1977 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
1978 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
1979 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
1980 default: return Register();
1981 }
1982}
1983
1984// FastEmit functions for ISD::CTPOP.
1985
1986Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
1987 if (RetVT.SimpleTy != MVT::v8i8)
1988 return Register();
1989 if ((Subtarget->hasNEON())) {
1990 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTd, RC: &ARM::DPRRegClass, Op0);
1991 }
1992 return Register();
1993}
1994
1995Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
1996 if (RetVT.SimpleTy != MVT::v16i8)
1997 return Register();
1998 if ((Subtarget->hasNEON())) {
1999 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTq, RC: &ARM::QPRRegClass, Op0);
2000 }
2001 return Register();
2002}
2003
2004Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2005 switch (VT.SimpleTy) {
2006 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2007 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2008 default: return Register();
2009 }
2010}
2011
2012// FastEmit functions for ISD::FABS.
2013
2014Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2015 if (RetVT.SimpleTy != MVT::f16)
2016 return Register();
2017 if ((Subtarget->hasFullFP16())) {
2018 return fastEmitInst_r(MachineInstOpcode: ARM::VABSH, RC: &ARM::HPRRegClass, Op0);
2019 }
2020 return Register();
2021}
2022
2023Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2024 if (RetVT.SimpleTy != MVT::f32)
2025 return Register();
2026 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2027 return fastEmitInst_r(MachineInstOpcode: ARM::VABSS, RC: &ARM::SPRRegClass, Op0);
2028 }
2029 return Register();
2030}
2031
2032Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2033 if (RetVT.SimpleTy != MVT::f64)
2034 return Register();
2035 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2036 return fastEmitInst_r(MachineInstOpcode: ARM::VABSD, RC: &ARM::DPRRegClass, Op0);
2037 }
2038 return Register();
2039}
2040
2041Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2042 if (RetVT.SimpleTy != MVT::v4f16)
2043 return Register();
2044 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2045 return fastEmitInst_r(MachineInstOpcode: ARM::VABShd, RC: &ARM::DPRRegClass, Op0);
2046 }
2047 return Register();
2048}
2049
2050Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2051 if (RetVT.SimpleTy != MVT::v8f16)
2052 return Register();
2053 if ((Subtarget->hasMVEIntegerOps())) {
2054 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf16, RC: &ARM::MQPRRegClass, Op0);
2055 }
2056 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2057 return fastEmitInst_r(MachineInstOpcode: ARM::VABShq, RC: &ARM::QPRRegClass, Op0);
2058 }
2059 return Register();
2060}
2061
2062Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2063 if (RetVT.SimpleTy != MVT::v2f32)
2064 return Register();
2065 if ((Subtarget->hasNEON())) {
2066 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfd, RC: &ARM::DPRRegClass, Op0);
2067 }
2068 return Register();
2069}
2070
2071Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2072 if (RetVT.SimpleTy != MVT::v4f32)
2073 return Register();
2074 if ((Subtarget->hasMVEIntegerOps())) {
2075 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf32, RC: &ARM::MQPRRegClass, Op0);
2076 }
2077 if ((Subtarget->hasNEON())) {
2078 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfq, RC: &ARM::QPRRegClass, Op0);
2079 }
2080 return Register();
2081}
2082
2083Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2084 switch (VT.SimpleTy) {
2085 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2086 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2087 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2088 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2089 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2090 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2091 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2092 default: return Register();
2093 }
2094}
2095
2096// FastEmit functions for ISD::FCEIL.
2097
2098Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2099 if (RetVT.SimpleTy != MVT::f16)
2100 return Register();
2101 if ((Subtarget->hasFullFP16())) {
2102 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
2103 }
2104 return Register();
2105}
2106
2107Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2108 if (RetVT.SimpleTy != MVT::f32)
2109 return Register();
2110 if ((Subtarget->hasFPARMv8Base())) {
2111 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
2112 }
2113 return Register();
2114}
2115
2116Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2117 if (RetVT.SimpleTy != MVT::f64)
2118 return Register();
2119 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2120 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
2121 }
2122 return Register();
2123}
2124
2125Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2126 if (RetVT.SimpleTy != MVT::v4f16)
2127 return Register();
2128 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2129 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDh, RC: &ARM::DPRRegClass, Op0);
2130 }
2131 return Register();
2132}
2133
2134Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2135 if (RetVT.SimpleTy != MVT::v8f16)
2136 return Register();
2137 if ((Subtarget->hasMVEFloatOps())) {
2138 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
2139 }
2140 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2141 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQh, RC: &ARM::QPRRegClass, Op0);
2142 }
2143 return Register();
2144}
2145
2146Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2147 if (RetVT.SimpleTy != MVT::v2f32)
2148 return Register();
2149 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2150 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDf, RC: &ARM::DPRRegClass, Op0);
2151 }
2152 return Register();
2153}
2154
2155Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2156 if (RetVT.SimpleTy != MVT::v4f32)
2157 return Register();
2158 if ((Subtarget->hasMVEFloatOps())) {
2159 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
2160 }
2161 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2162 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQf, RC: &ARM::QPRRegClass, Op0);
2163 }
2164 return Register();
2165}
2166
2167Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2168 switch (VT.SimpleTy) {
2169 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2170 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2171 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2172 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2173 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2174 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2175 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2176 default: return Register();
2177 }
2178}
2179
2180// FastEmit functions for ISD::FFLOOR.
2181
2182Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2183 if (RetVT.SimpleTy != MVT::f16)
2184 return Register();
2185 if ((Subtarget->hasFullFP16())) {
2186 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
2187 }
2188 return Register();
2189}
2190
2191Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2192 if (RetVT.SimpleTy != MVT::f32)
2193 return Register();
2194 if ((Subtarget->hasFPARMv8Base())) {
2195 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
2196 }
2197 return Register();
2198}
2199
2200Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2201 if (RetVT.SimpleTy != MVT::f64)
2202 return Register();
2203 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2204 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
2205 }
2206 return Register();
2207}
2208
2209Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2210 if (RetVT.SimpleTy != MVT::v4f16)
2211 return Register();
2212 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2213 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDh, RC: &ARM::DPRRegClass, Op0);
2214 }
2215 return Register();
2216}
2217
2218Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2219 if (RetVT.SimpleTy != MVT::v8f16)
2220 return Register();
2221 if ((Subtarget->hasMVEFloatOps())) {
2222 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
2223 }
2224 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2225 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQh, RC: &ARM::QPRRegClass, Op0);
2226 }
2227 return Register();
2228}
2229
2230Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2231 if (RetVT.SimpleTy != MVT::v2f32)
2232 return Register();
2233 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2234 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDf, RC: &ARM::DPRRegClass, Op0);
2235 }
2236 return Register();
2237}
2238
2239Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2240 if (RetVT.SimpleTy != MVT::v4f32)
2241 return Register();
2242 if ((Subtarget->hasMVEFloatOps())) {
2243 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
2244 }
2245 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2246 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQf, RC: &ARM::QPRRegClass, Op0);
2247 }
2248 return Register();
2249}
2250
2251Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2252 switch (VT.SimpleTy) {
2253 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2254 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2255 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2256 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2257 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2258 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2259 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2260 default: return Register();
2261 }
2262}
2263
2264// FastEmit functions for ISD::FNEARBYINT.
2265
2266Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2267 if (RetVT.SimpleTy != MVT::f16)
2268 return Register();
2269 if ((Subtarget->hasFullFP16())) {
2270 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
2271 }
2272 return Register();
2273}
2274
2275Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2276 if (RetVT.SimpleTy != MVT::f32)
2277 return Register();
2278 if ((Subtarget->hasFPARMv8Base())) {
2279 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
2280 }
2281 return Register();
2282}
2283
2284Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2285 if (RetVT.SimpleTy != MVT::f64)
2286 return Register();
2287 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2288 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
2289 }
2290 return Register();
2291}
2292
2293Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2294 switch (VT.SimpleTy) {
2295 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2296 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2297 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2298 default: return Register();
2299 }
2300}
2301
2302// FastEmit functions for ISD::FNEG.
2303
2304Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2305 if (RetVT.SimpleTy != MVT::f16)
2306 return Register();
2307 if ((Subtarget->hasFullFP16())) {
2308 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGH, RC: &ARM::HPRRegClass, Op0);
2309 }
2310 return Register();
2311}
2312
2313Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2314 if (RetVT.SimpleTy != MVT::f32)
2315 return Register();
2316 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2317 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGS, RC: &ARM::SPRRegClass, Op0);
2318 }
2319 return Register();
2320}
2321
2322Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2323 if (RetVT.SimpleTy != MVT::f64)
2324 return Register();
2325 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2326 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGD, RC: &ARM::DPRRegClass, Op0);
2327 }
2328 return Register();
2329}
2330
2331Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2332 if (RetVT.SimpleTy != MVT::v4f16)
2333 return Register();
2334 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2335 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhd, RC: &ARM::DPRRegClass, Op0);
2336 }
2337 return Register();
2338}
2339
2340Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
2341 if (RetVT.SimpleTy != MVT::v8f16)
2342 return Register();
2343 if ((Subtarget->hasMVEIntegerOps())) {
2344 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf16, RC: &ARM::MQPRRegClass, Op0);
2345 }
2346 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2347 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhq, RC: &ARM::QPRRegClass, Op0);
2348 }
2349 return Register();
2350}
2351
2352Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
2353 if (RetVT.SimpleTy != MVT::v2f32)
2354 return Register();
2355 if ((Subtarget->hasNEON())) {
2356 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGfd, RC: &ARM::DPRRegClass, Op0);
2357 }
2358 return Register();
2359}
2360
2361Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
2362 if (RetVT.SimpleTy != MVT::v4f32)
2363 return Register();
2364 if ((Subtarget->hasMVEIntegerOps())) {
2365 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf32, RC: &ARM::MQPRRegClass, Op0);
2366 }
2367 if ((Subtarget->hasNEON())) {
2368 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGf32q, RC: &ARM::QPRRegClass, Op0);
2369 }
2370 return Register();
2371}
2372
2373Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
2374 switch (VT.SimpleTy) {
2375 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
2376 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
2377 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
2378 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
2379 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
2380 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
2381 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
2382 default: return Register();
2383 }
2384}
2385
2386// FastEmit functions for ISD::FP_EXTEND.
2387
2388Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2389 if (RetVT.SimpleTy != MVT::f64)
2390 return Register();
2391 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2392 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
2393 }
2394 return Register();
2395}
2396
2397Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2398 if (RetVT.SimpleTy != MVT::v4f32)
2399 return Register();
2400 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2f, RC: &ARM::QPRRegClass, Op0);
2401}
2402
2403Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2404 switch (VT.SimpleTy) {
2405 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2406 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
2407 default: return Register();
2408 }
2409}
2410
2411// FastEmit functions for ISD::FP_ROUND.
2412
2413Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2414 if (RetVT.SimpleTy != MVT::f32)
2415 return Register();
2416 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2417 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
2418 }
2419 return Register();
2420}
2421
2422Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2423 if (RetVT.SimpleTy != MVT::v4f16)
2424 return Register();
2425 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2h, RC: &ARM::DPRRegClass, Op0);
2426}
2427
2428Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2429 switch (VT.SimpleTy) {
2430 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
2431 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
2432 default: return Register();
2433 }
2434}
2435
2436// FastEmit functions for ISD::FP_TO_SINT.
2437
2438Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2439 if (RetVT.SimpleTy != MVT::v4i16)
2440 return Register();
2441 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2442 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sd, RC: &ARM::DPRRegClass, Op0);
2443 }
2444 return Register();
2445}
2446
2447Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2448 if (RetVT.SimpleTy != MVT::v8i16)
2449 return Register();
2450 if ((Subtarget->hasMVEFloatOps())) {
2451 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs16f16z, RC: &ARM::MQPRRegClass, Op0);
2452 }
2453 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2454 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sq, RC: &ARM::QPRRegClass, Op0);
2455 }
2456 return Register();
2457}
2458
2459Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2460 if (RetVT.SimpleTy != MVT::v2i32)
2461 return Register();
2462 if ((Subtarget->hasNEON())) {
2463 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sd, RC: &ARM::DPRRegClass, Op0);
2464 }
2465 return Register();
2466}
2467
2468Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2469 if (RetVT.SimpleTy != MVT::v4i32)
2470 return Register();
2471 if ((Subtarget->hasMVEFloatOps())) {
2472 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs32f32z, RC: &ARM::MQPRRegClass, Op0);
2473 }
2474 if ((Subtarget->hasNEON())) {
2475 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sq, RC: &ARM::QPRRegClass, Op0);
2476 }
2477 return Register();
2478}
2479
2480Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2481 switch (VT.SimpleTy) {
2482 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
2483 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
2484 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
2485 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
2486 default: return Register();
2487 }
2488}
2489
2490// FastEmit functions for ISD::FP_TO_UINT.
2491
2492Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2493 if (RetVT.SimpleTy != MVT::v4i16)
2494 return Register();
2495 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2496 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2ud, RC: &ARM::DPRRegClass, Op0);
2497 }
2498 return Register();
2499}
2500
2501Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2502 if (RetVT.SimpleTy != MVT::v8i16)
2503 return Register();
2504 if ((Subtarget->hasMVEFloatOps())) {
2505 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu16f16z, RC: &ARM::MQPRRegClass, Op0);
2506 }
2507 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2508 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2uq, RC: &ARM::QPRRegClass, Op0);
2509 }
2510 return Register();
2511}
2512
2513Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2514 if (RetVT.SimpleTy != MVT::v2i32)
2515 return Register();
2516 if ((Subtarget->hasNEON())) {
2517 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2ud, RC: &ARM::DPRRegClass, Op0);
2518 }
2519 return Register();
2520}
2521
2522Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2523 if (RetVT.SimpleTy != MVT::v4i32)
2524 return Register();
2525 if ((Subtarget->hasMVEFloatOps())) {
2526 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu32f32z, RC: &ARM::MQPRRegClass, Op0);
2527 }
2528 if ((Subtarget->hasNEON())) {
2529 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2uq, RC: &ARM::QPRRegClass, Op0);
2530 }
2531 return Register();
2532}
2533
2534Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2535 switch (VT.SimpleTy) {
2536 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
2537 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
2538 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
2539 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
2540 default: return Register();
2541 }
2542}
2543
2544// FastEmit functions for ISD::FRINT.
2545
2546Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
2547 if (RetVT.SimpleTy != MVT::f16)
2548 return Register();
2549 if ((Subtarget->hasFullFP16())) {
2550 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
2551 }
2552 return Register();
2553}
2554
2555Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
2556 if (RetVT.SimpleTy != MVT::f32)
2557 return Register();
2558 if ((Subtarget->hasFPARMv8Base())) {
2559 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
2560 }
2561 return Register();
2562}
2563
2564Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
2565 if (RetVT.SimpleTy != MVT::f64)
2566 return Register();
2567 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2568 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
2569 }
2570 return Register();
2571}
2572
2573Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2574 if (RetVT.SimpleTy != MVT::v4f16)
2575 return Register();
2576 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2577 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDh, RC: &ARM::DPRRegClass, Op0);
2578 }
2579 return Register();
2580}
2581
2582Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2583 if (RetVT.SimpleTy != MVT::v8f16)
2584 return Register();
2585 if ((Subtarget->hasMVEFloatOps())) {
2586 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16X, RC: &ARM::MQPRRegClass, Op0);
2587 }
2588 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2589 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQh, RC: &ARM::QPRRegClass, Op0);
2590 }
2591 return Register();
2592}
2593
2594Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2595 if (RetVT.SimpleTy != MVT::v2f32)
2596 return Register();
2597 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2598 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDf, RC: &ARM::DPRRegClass, Op0);
2599 }
2600 return Register();
2601}
2602
2603Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2604 if (RetVT.SimpleTy != MVT::v4f32)
2605 return Register();
2606 if ((Subtarget->hasMVEFloatOps())) {
2607 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32X, RC: &ARM::MQPRRegClass, Op0);
2608 }
2609 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2610 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQf, RC: &ARM::QPRRegClass, Op0);
2611 }
2612 return Register();
2613}
2614
2615Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
2616 switch (VT.SimpleTy) {
2617 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
2618 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
2619 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
2620 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
2621 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
2622 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
2623 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
2624 default: return Register();
2625 }
2626}
2627
2628// FastEmit functions for ISD::FROUND.
2629
2630Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
2631 if (RetVT.SimpleTy != MVT::f16)
2632 return Register();
2633 if ((Subtarget->hasFullFP16())) {
2634 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
2635 }
2636 return Register();
2637}
2638
2639Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
2640 if (RetVT.SimpleTy != MVT::f32)
2641 return Register();
2642 if ((Subtarget->hasFPARMv8Base())) {
2643 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
2644 }
2645 return Register();
2646}
2647
2648Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2649 if (RetVT.SimpleTy != MVT::f64)
2650 return Register();
2651 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2652 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
2653 }
2654 return Register();
2655}
2656
2657Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2658 if (RetVT.SimpleTy != MVT::v4f16)
2659 return Register();
2660 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2661 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDh, RC: &ARM::DPRRegClass, Op0);
2662 }
2663 return Register();
2664}
2665
2666Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2667 if (RetVT.SimpleTy != MVT::v8f16)
2668 return Register();
2669 if ((Subtarget->hasMVEFloatOps())) {
2670 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
2671 }
2672 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2673 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQh, RC: &ARM::QPRRegClass, Op0);
2674 }
2675 return Register();
2676}
2677
2678Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
2679 if (RetVT.SimpleTy != MVT::v2f32)
2680 return Register();
2681 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2682 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDf, RC: &ARM::DPRRegClass, Op0);
2683 }
2684 return Register();
2685}
2686
2687Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2688 if (RetVT.SimpleTy != MVT::v4f32)
2689 return Register();
2690 if ((Subtarget->hasMVEFloatOps())) {
2691 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
2692 }
2693 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2694 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQf, RC: &ARM::QPRRegClass, Op0);
2695 }
2696 return Register();
2697}
2698
2699Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
2700 switch (VT.SimpleTy) {
2701 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
2702 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
2703 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
2704 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
2705 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
2706 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
2707 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
2708 default: return Register();
2709 }
2710}
2711
2712// FastEmit functions for ISD::FROUNDEVEN.
2713
2714Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
2715 if (RetVT.SimpleTy != MVT::f16)
2716 return Register();
2717 if ((Subtarget->hasFullFP16())) {
2718 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
2719 }
2720 return Register();
2721}
2722
2723Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
2724 if (RetVT.SimpleTy != MVT::f32)
2725 return Register();
2726 if ((Subtarget->hasFPARMv8Base())) {
2727 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
2728 }
2729 return Register();
2730}
2731
2732Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
2733 if (RetVT.SimpleTy != MVT::f64)
2734 return Register();
2735 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2736 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
2737 }
2738 return Register();
2739}
2740
2741Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
2742 if (RetVT.SimpleTy != MVT::v4f16)
2743 return Register();
2744 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2745 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDh, RC: &ARM::DPRRegClass, Op0);
2746 }
2747 return Register();
2748}
2749
2750Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
2751 if (RetVT.SimpleTy != MVT::v8f16)
2752 return Register();
2753 if ((Subtarget->hasMVEFloatOps())) {
2754 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16N, RC: &ARM::MQPRRegClass, Op0);
2755 }
2756 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2757 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQh, RC: &ARM::QPRRegClass, Op0);
2758 }
2759 return Register();
2760}
2761
2762Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
2763 if (RetVT.SimpleTy != MVT::v2f32)
2764 return Register();
2765 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2766 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDf, RC: &ARM::DPRRegClass, Op0);
2767 }
2768 return Register();
2769}
2770
2771Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
2772 if (RetVT.SimpleTy != MVT::v4f32)
2773 return Register();
2774 if ((Subtarget->hasMVEFloatOps())) {
2775 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32N, RC: &ARM::MQPRRegClass, Op0);
2776 }
2777 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2778 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQf, RC: &ARM::QPRRegClass, Op0);
2779 }
2780 return Register();
2781}
2782
2783Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
2784 switch (VT.SimpleTy) {
2785 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
2786 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
2787 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
2788 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
2789 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
2790 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
2791 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
2792 default: return Register();
2793 }
2794}
2795
2796// FastEmit functions for ISD::FSQRT.
2797
2798Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
2799 if (RetVT.SimpleTy != MVT::f16)
2800 return Register();
2801 if ((Subtarget->hasFullFP16())) {
2802 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
2803 }
2804 return Register();
2805}
2806
2807Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
2808 if (RetVT.SimpleTy != MVT::f32)
2809 return Register();
2810 if ((Subtarget->hasVFP2Base())) {
2811 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
2812 }
2813 return Register();
2814}
2815
2816Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
2817 if (RetVT.SimpleTy != MVT::f64)
2818 return Register();
2819 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2820 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
2821 }
2822 return Register();
2823}
2824
2825Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
2826 switch (VT.SimpleTy) {
2827 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
2828 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
2829 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
2830 default: return Register();
2831 }
2832}
2833
2834// FastEmit functions for ISD::FTRUNC.
2835
2836Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
2837 if (RetVT.SimpleTy != MVT::f16)
2838 return Register();
2839 if ((Subtarget->hasFullFP16())) {
2840 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
2841 }
2842 return Register();
2843}
2844
2845Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
2846 if (RetVT.SimpleTy != MVT::f32)
2847 return Register();
2848 if ((Subtarget->hasFPARMv8Base())) {
2849 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
2850 }
2851 return Register();
2852}
2853
2854Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
2855 if (RetVT.SimpleTy != MVT::f64)
2856 return Register();
2857 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2858 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
2859 }
2860 return Register();
2861}
2862
2863Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
2864 if (RetVT.SimpleTy != MVT::v4f16)
2865 return Register();
2866 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2867 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDh, RC: &ARM::DPRRegClass, Op0);
2868 }
2869 return Register();
2870}
2871
2872Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
2873 if (RetVT.SimpleTy != MVT::v8f16)
2874 return Register();
2875 if ((Subtarget->hasMVEFloatOps())) {
2876 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
2877 }
2878 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2879 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQh, RC: &ARM::QPRRegClass, Op0);
2880 }
2881 return Register();
2882}
2883
2884Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
2885 if (RetVT.SimpleTy != MVT::v2f32)
2886 return Register();
2887 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2888 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDf, RC: &ARM::DPRRegClass, Op0);
2889 }
2890 return Register();
2891}
2892
2893Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
2894 if (RetVT.SimpleTy != MVT::v4f32)
2895 return Register();
2896 if ((Subtarget->hasMVEFloatOps())) {
2897 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
2898 }
2899 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2900 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQf, RC: &ARM::QPRRegClass, Op0);
2901 }
2902 return Register();
2903}
2904
2905Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
2906 switch (VT.SimpleTy) {
2907 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
2908 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
2909 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
2910 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
2911 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
2912 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
2913 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
2914 default: return Register();
2915 }
2916}
2917
2918// FastEmit functions for ISD::SIGN_EXTEND.
2919
2920Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
2921 if (RetVT.SimpleTy != MVT::v8i16)
2922 return Register();
2923 if ((Subtarget->hasNEON())) {
2924 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv8i16, RC: &ARM::QPRRegClass, Op0);
2925 }
2926 return Register();
2927}
2928
2929Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
2930 if (RetVT.SimpleTy != MVT::v4i32)
2931 return Register();
2932 if ((Subtarget->hasNEON())) {
2933 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv4i32, RC: &ARM::QPRRegClass, Op0);
2934 }
2935 return Register();
2936}
2937
2938Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
2939 if (RetVT.SimpleTy != MVT::v2i64)
2940 return Register();
2941 if ((Subtarget->hasNEON())) {
2942 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv2i64, RC: &ARM::QPRRegClass, Op0);
2943 }
2944 return Register();
2945}
2946
2947Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2948 switch (VT.SimpleTy) {
2949 case MVT::v8i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(RetVT, Op0);
2950 case MVT::v4i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(RetVT, Op0);
2951 case MVT::v2i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(RetVT, Op0);
2952 default: return Register();
2953 }
2954}
2955
2956// FastEmit functions for ISD::SINT_TO_FP.
2957
2958Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2959 if (RetVT.SimpleTy != MVT::v4f16)
2960 return Register();
2961 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2962 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hd, RC: &ARM::DPRRegClass, Op0);
2963 }
2964 return Register();
2965}
2966
2967Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2968 if (RetVT.SimpleTy != MVT::v8f16)
2969 return Register();
2970 if ((Subtarget->hasMVEFloatOps())) {
2971 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16s16n, RC: &ARM::MQPRRegClass, Op0);
2972 }
2973 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2974 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hq, RC: &ARM::QPRRegClass, Op0);
2975 }
2976 return Register();
2977}
2978
2979Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2980 if (RetVT.SimpleTy != MVT::v2f32)
2981 return Register();
2982 if ((Subtarget->hasNEON())) {
2983 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fd, RC: &ARM::DPRRegClass, Op0);
2984 }
2985 return Register();
2986}
2987
2988Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2989 if (RetVT.SimpleTy != MVT::v4f32)
2990 return Register();
2991 if ((Subtarget->hasMVEFloatOps())) {
2992 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32s32n, RC: &ARM::MQPRRegClass, Op0);
2993 }
2994 if ((Subtarget->hasNEON())) {
2995 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fq, RC: &ARM::QPRRegClass, Op0);
2996 }
2997 return Register();
2998}
2999
3000Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3001 switch (VT.SimpleTy) {
3002 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3003 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3004 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3005 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3006 default: return Register();
3007 }
3008}
3009
3010// FastEmit functions for ISD::STRICT_FCEIL.
3011
3012Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
3013 if (RetVT.SimpleTy != MVT::f16)
3014 return Register();
3015 if ((Subtarget->hasFullFP16())) {
3016 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
3017 }
3018 return Register();
3019}
3020
3021Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
3022 if (RetVT.SimpleTy != MVT::f32)
3023 return Register();
3024 if ((Subtarget->hasFPARMv8Base())) {
3025 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
3026 }
3027 return Register();
3028}
3029
3030Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
3031 if (RetVT.SimpleTy != MVT::f64)
3032 return Register();
3033 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3034 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
3035 }
3036 return Register();
3037}
3038
3039Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
3040 if (RetVT.SimpleTy != MVT::v4f16)
3041 return Register();
3042 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3043 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDh, RC: &ARM::DPRRegClass, Op0);
3044 }
3045 return Register();
3046}
3047
3048Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
3049 if (RetVT.SimpleTy != MVT::v8f16)
3050 return Register();
3051 if ((Subtarget->hasMVEFloatOps())) {
3052 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
3053 }
3054 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3055 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQh, RC: &ARM::QPRRegClass, Op0);
3056 }
3057 return Register();
3058}
3059
3060Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
3061 if (RetVT.SimpleTy != MVT::v2f32)
3062 return Register();
3063 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3064 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDf, RC: &ARM::DPRRegClass, Op0);
3065 }
3066 return Register();
3067}
3068
3069Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
3070 if (RetVT.SimpleTy != MVT::v4f32)
3071 return Register();
3072 if ((Subtarget->hasMVEFloatOps())) {
3073 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
3074 }
3075 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3076 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQf, RC: &ARM::QPRRegClass, Op0);
3077 }
3078 return Register();
3079}
3080
3081Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
3082 switch (VT.SimpleTy) {
3083 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
3084 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
3085 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
3086 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
3087 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
3088 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
3089 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
3090 default: return Register();
3091 }
3092}
3093
3094// FastEmit functions for ISD::STRICT_FFLOOR.
3095
3096Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
3097 if (RetVT.SimpleTy != MVT::f16)
3098 return Register();
3099 if ((Subtarget->hasFullFP16())) {
3100 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
3101 }
3102 return Register();
3103}
3104
3105Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
3106 if (RetVT.SimpleTy != MVT::f32)
3107 return Register();
3108 if ((Subtarget->hasFPARMv8Base())) {
3109 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
3110 }
3111 return Register();
3112}
3113
3114Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
3115 if (RetVT.SimpleTy != MVT::f64)
3116 return Register();
3117 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3118 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
3119 }
3120 return Register();
3121}
3122
3123Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
3124 if (RetVT.SimpleTy != MVT::v4f16)
3125 return Register();
3126 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3127 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDh, RC: &ARM::DPRRegClass, Op0);
3128 }
3129 return Register();
3130}
3131
3132Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
3133 if (RetVT.SimpleTy != MVT::v8f16)
3134 return Register();
3135 if ((Subtarget->hasMVEFloatOps())) {
3136 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
3137 }
3138 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3139 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQh, RC: &ARM::QPRRegClass, Op0);
3140 }
3141 return Register();
3142}
3143
3144Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
3145 if (RetVT.SimpleTy != MVT::v2f32)
3146 return Register();
3147 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3148 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDf, RC: &ARM::DPRRegClass, Op0);
3149 }
3150 return Register();
3151}
3152
3153Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
3154 if (RetVT.SimpleTy != MVT::v4f32)
3155 return Register();
3156 if ((Subtarget->hasMVEFloatOps())) {
3157 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
3158 }
3159 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3160 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQf, RC: &ARM::QPRRegClass, Op0);
3161 }
3162 return Register();
3163}
3164
3165Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
3166 switch (VT.SimpleTy) {
3167 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
3168 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
3169 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
3170 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
3171 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
3172 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
3173 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
3174 default: return Register();
3175 }
3176}
3177
3178// FastEmit functions for ISD::STRICT_FNEARBYINT.
3179
3180Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
3181 if (RetVT.SimpleTy != MVT::f16)
3182 return Register();
3183 if ((Subtarget->hasFullFP16())) {
3184 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
3185 }
3186 return Register();
3187}
3188
3189Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
3190 if (RetVT.SimpleTy != MVT::f32)
3191 return Register();
3192 if ((Subtarget->hasFPARMv8Base())) {
3193 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
3194 }
3195 return Register();
3196}
3197
3198Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
3199 if (RetVT.SimpleTy != MVT::f64)
3200 return Register();
3201 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3202 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
3203 }
3204 return Register();
3205}
3206
3207Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
3208 switch (VT.SimpleTy) {
3209 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
3210 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
3211 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
3212 default: return Register();
3213 }
3214}
3215
3216// FastEmit functions for ISD::STRICT_FP_EXTEND.
3217
3218Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3219 if (RetVT.SimpleTy != MVT::f64)
3220 return Register();
3221 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3222 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
3223 }
3224 return Register();
3225}
3226
3227Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3228 switch (VT.SimpleTy) {
3229 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3230 default: return Register();
3231 }
3232}
3233
3234// FastEmit functions for ISD::STRICT_FP_ROUND.
3235
3236Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3237 if (RetVT.SimpleTy != MVT::f32)
3238 return Register();
3239 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3240 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
3241 }
3242 return Register();
3243}
3244
3245Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3246 switch (VT.SimpleTy) {
3247 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
3248 default: return Register();
3249 }
3250}
3251
3252// FastEmit functions for ISD::STRICT_FRINT.
3253
3254Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3255 if (RetVT.SimpleTy != MVT::f16)
3256 return Register();
3257 if ((Subtarget->hasFullFP16())) {
3258 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
3259 }
3260 return Register();
3261}
3262
3263Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3264 if (RetVT.SimpleTy != MVT::f32)
3265 return Register();
3266 if ((Subtarget->hasFPARMv8Base())) {
3267 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
3268 }
3269 return Register();
3270}
3271
3272Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3273 if (RetVT.SimpleTy != MVT::f64)
3274 return Register();
3275 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3276 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
3277 }
3278 return Register();
3279}
3280
3281Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3282 if (RetVT.SimpleTy != MVT::v4f16)
3283 return Register();
3284 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3285 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDh, RC: &ARM::DPRRegClass, Op0);
3286 }
3287 return Register();
3288}
3289
3290Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3291 if (RetVT.SimpleTy != MVT::v8f16)
3292 return Register();
3293 if ((Subtarget->hasMVEFloatOps())) {
3294 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16X, RC: &ARM::MQPRRegClass, Op0);
3295 }
3296 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3297 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQh, RC: &ARM::QPRRegClass, Op0);
3298 }
3299 return Register();
3300}
3301
3302Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3303 if (RetVT.SimpleTy != MVT::v2f32)
3304 return Register();
3305 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3306 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDf, RC: &ARM::DPRRegClass, Op0);
3307 }
3308 return Register();
3309}
3310
3311Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3312 if (RetVT.SimpleTy != MVT::v4f32)
3313 return Register();
3314 if ((Subtarget->hasMVEFloatOps())) {
3315 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32X, RC: &ARM::MQPRRegClass, Op0);
3316 }
3317 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3318 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQf, RC: &ARM::QPRRegClass, Op0);
3319 }
3320 return Register();
3321}
3322
3323Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3324 switch (VT.SimpleTy) {
3325 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
3326 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
3327 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
3328 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
3329 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
3330 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
3331 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
3332 default: return Register();
3333 }
3334}
3335
3336// FastEmit functions for ISD::STRICT_FROUND.
3337
3338Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3339 if (RetVT.SimpleTy != MVT::f16)
3340 return Register();
3341 if ((Subtarget->hasFullFP16())) {
3342 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
3343 }
3344 return Register();
3345}
3346
3347Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3348 if (RetVT.SimpleTy != MVT::f32)
3349 return Register();
3350 if ((Subtarget->hasFPARMv8Base())) {
3351 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
3352 }
3353 return Register();
3354}
3355
3356Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3357 if (RetVT.SimpleTy != MVT::f64)
3358 return Register();
3359 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3360 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
3361 }
3362 return Register();
3363}
3364
3365Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3366 if (RetVT.SimpleTy != MVT::v4f16)
3367 return Register();
3368 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3369 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDh, RC: &ARM::DPRRegClass, Op0);
3370 }
3371 return Register();
3372}
3373
3374Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3375 if (RetVT.SimpleTy != MVT::v8f16)
3376 return Register();
3377 if ((Subtarget->hasMVEFloatOps())) {
3378 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
3379 }
3380 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3381 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQh, RC: &ARM::QPRRegClass, Op0);
3382 }
3383 return Register();
3384}
3385
3386Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3387 if (RetVT.SimpleTy != MVT::v2f32)
3388 return Register();
3389 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3390 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDf, RC: &ARM::DPRRegClass, Op0);
3391 }
3392 return Register();
3393}
3394
3395Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3396 if (RetVT.SimpleTy != MVT::v4f32)
3397 return Register();
3398 if ((Subtarget->hasMVEFloatOps())) {
3399 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
3400 }
3401 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3402 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQf, RC: &ARM::QPRRegClass, Op0);
3403 }
3404 return Register();
3405}
3406
3407Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3408 switch (VT.SimpleTy) {
3409 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
3410 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
3411 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
3412 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
3413 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
3414 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
3415 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
3416 default: return Register();
3417 }
3418}
3419
3420// FastEmit functions for ISD::STRICT_FROUNDEVEN.
3421
3422Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3423 if (RetVT.SimpleTy != MVT::f16)
3424 return Register();
3425 if ((Subtarget->hasFullFP16())) {
3426 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
3427 }
3428 return Register();
3429}
3430
3431Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3432 if (RetVT.SimpleTy != MVT::f32)
3433 return Register();
3434 if ((Subtarget->hasFPARMv8Base())) {
3435 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
3436 }
3437 return Register();
3438}
3439
3440Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3441 if (RetVT.SimpleTy != MVT::f64)
3442 return Register();
3443 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3444 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
3445 }
3446 return Register();
3447}
3448
3449Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3450 if (RetVT.SimpleTy != MVT::v4f16)
3451 return Register();
3452 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3453 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDh, RC: &ARM::DPRRegClass, Op0);
3454 }
3455 return Register();
3456}
3457
3458Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3459 if (RetVT.SimpleTy != MVT::v8f16)
3460 return Register();
3461 if ((Subtarget->hasMVEFloatOps())) {
3462 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16N, RC: &ARM::MQPRRegClass, Op0);
3463 }
3464 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3465 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQh, RC: &ARM::QPRRegClass, Op0);
3466 }
3467 return Register();
3468}
3469
3470Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3471 if (RetVT.SimpleTy != MVT::v2f32)
3472 return Register();
3473 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3474 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDf, RC: &ARM::DPRRegClass, Op0);
3475 }
3476 return Register();
3477}
3478
3479Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3480 if (RetVT.SimpleTy != MVT::v4f32)
3481 return Register();
3482 if ((Subtarget->hasMVEFloatOps())) {
3483 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32N, RC: &ARM::MQPRRegClass, Op0);
3484 }
3485 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3486 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQf, RC: &ARM::QPRRegClass, Op0);
3487 }
3488 return Register();
3489}
3490
3491Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3492 switch (VT.SimpleTy) {
3493 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3494 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3495 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3496 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3497 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3498 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3499 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3500 default: return Register();
3501 }
3502}
3503
3504// FastEmit functions for ISD::STRICT_FSQRT.
3505
3506Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3507 if (RetVT.SimpleTy != MVT::f16)
3508 return Register();
3509 if ((Subtarget->hasFullFP16())) {
3510 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
3511 }
3512 return Register();
3513}
3514
3515Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3516 if (RetVT.SimpleTy != MVT::f32)
3517 return Register();
3518 if ((Subtarget->hasVFP2Base())) {
3519 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
3520 }
3521 return Register();
3522}
3523
3524Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3525 if (RetVT.SimpleTy != MVT::f64)
3526 return Register();
3527 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3528 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
3529 }
3530 return Register();
3531}
3532
3533Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3534 switch (VT.SimpleTy) {
3535 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
3536 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
3537 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
3538 default: return Register();
3539 }
3540}
3541
3542// FastEmit functions for ISD::STRICT_FTRUNC.
3543
3544Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3545 if (RetVT.SimpleTy != MVT::f16)
3546 return Register();
3547 if ((Subtarget->hasFullFP16())) {
3548 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
3549 }
3550 return Register();
3551}
3552
3553Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3554 if (RetVT.SimpleTy != MVT::f32)
3555 return Register();
3556 if ((Subtarget->hasFPARMv8Base())) {
3557 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
3558 }
3559 return Register();
3560}
3561
3562Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3563 if (RetVT.SimpleTy != MVT::f64)
3564 return Register();
3565 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3566 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
3567 }
3568 return Register();
3569}
3570
3571Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3572 if (RetVT.SimpleTy != MVT::v4f16)
3573 return Register();
3574 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3575 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDh, RC: &ARM::DPRRegClass, Op0);
3576 }
3577 return Register();
3578}
3579
3580Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3581 if (RetVT.SimpleTy != MVT::v8f16)
3582 return Register();
3583 if ((Subtarget->hasMVEFloatOps())) {
3584 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
3585 }
3586 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3587 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQh, RC: &ARM::QPRRegClass, Op0);
3588 }
3589 return Register();
3590}
3591
3592Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3593 if (RetVT.SimpleTy != MVT::v2f32)
3594 return Register();
3595 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3596 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDf, RC: &ARM::DPRRegClass, Op0);
3597 }
3598 return Register();
3599}
3600
3601Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3602 if (RetVT.SimpleTy != MVT::v4f32)
3603 return Register();
3604 if ((Subtarget->hasMVEFloatOps())) {
3605 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
3606 }
3607 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3608 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQf, RC: &ARM::QPRRegClass, Op0);
3609 }
3610 return Register();
3611}
3612
3613Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3614 switch (VT.SimpleTy) {
3615 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
3616 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
3617 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
3618 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3619 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3620 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3621 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3622 default: return Register();
3623 }
3624}
3625
3626// FastEmit functions for ISD::TRUNCATE.
3627
3628Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
3629 if (RetVT.SimpleTy != MVT::v8i8)
3630 return Register();
3631 if ((Subtarget->hasNEON())) {
3632 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv8i8, RC: &ARM::DPRRegClass, Op0);
3633 }
3634 return Register();
3635}
3636
3637Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
3638 if (RetVT.SimpleTy != MVT::v4i16)
3639 return Register();
3640 if ((Subtarget->hasNEON())) {
3641 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv4i16, RC: &ARM::DPRRegClass, Op0);
3642 }
3643 return Register();
3644}
3645
3646Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
3647 if (RetVT.SimpleTy != MVT::v2i32)
3648 return Register();
3649 if ((Subtarget->hasNEON())) {
3650 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv2i32, RC: &ARM::DPRRegClass, Op0);
3651 }
3652 return Register();
3653}
3654
3655Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3656 switch (VT.SimpleTy) {
3657 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
3658 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
3659 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
3660 default: return Register();
3661 }
3662}
3663
3664// FastEmit functions for ISD::UINT_TO_FP.
3665
3666Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
3667 if (RetVT.SimpleTy != MVT::v4f16)
3668 return Register();
3669 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3670 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hd, RC: &ARM::DPRRegClass, Op0);
3671 }
3672 return Register();
3673}
3674
3675Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3676 if (RetVT.SimpleTy != MVT::v8f16)
3677 return Register();
3678 if ((Subtarget->hasMVEFloatOps())) {
3679 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16u16n, RC: &ARM::MQPRRegClass, Op0);
3680 }
3681 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3682 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hq, RC: &ARM::QPRRegClass, Op0);
3683 }
3684 return Register();
3685}
3686
3687Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
3688 if (RetVT.SimpleTy != MVT::v2f32)
3689 return Register();
3690 if ((Subtarget->hasNEON())) {
3691 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fd, RC: &ARM::DPRRegClass, Op0);
3692 }
3693 return Register();
3694}
3695
3696Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3697 if (RetVT.SimpleTy != MVT::v4f32)
3698 return Register();
3699 if ((Subtarget->hasMVEFloatOps())) {
3700 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32u32n, RC: &ARM::MQPRRegClass, Op0);
3701 }
3702 if ((Subtarget->hasNEON())) {
3703 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fq, RC: &ARM::QPRRegClass, Op0);
3704 }
3705 return Register();
3706}
3707
3708Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3709 switch (VT.SimpleTy) {
3710 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3711 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3712 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3713 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3714 default: return Register();
3715 }
3716}
3717
3718// FastEmit functions for ISD::VECREDUCE_ADD.
3719
3720Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
3721 if (RetVT.SimpleTy != MVT::i32)
3722 return Register();
3723 if ((Subtarget->hasMVEIntegerOps())) {
3724 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3725 }
3726 return Register();
3727}
3728
3729Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
3730 if (RetVT.SimpleTy != MVT::i32)
3731 return Register();
3732 if ((Subtarget->hasMVEIntegerOps())) {
3733 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3734 }
3735 return Register();
3736}
3737
3738Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
3739 if (RetVT.SimpleTy != MVT::i32)
3740 return Register();
3741 if ((Subtarget->hasMVEIntegerOps())) {
3742 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3743 }
3744 return Register();
3745}
3746
3747Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
3748 switch (VT.SimpleTy) {
3749 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
3750 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
3751 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
3752 default: return Register();
3753 }
3754}
3755
3756// FastEmit functions for ISD::ZERO_EXTEND.
3757
3758Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
3759 if (RetVT.SimpleTy != MVT::v8i16)
3760 return Register();
3761 if ((Subtarget->hasNEON())) {
3762 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
3763 }
3764 return Register();
3765}
3766
3767Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
3768 if (RetVT.SimpleTy != MVT::v4i32)
3769 return Register();
3770 if ((Subtarget->hasNEON())) {
3771 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
3772 }
3773 return Register();
3774}
3775
3776Register fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
3777 if (RetVT.SimpleTy != MVT::v2i64)
3778 return Register();
3779 if ((Subtarget->hasNEON())) {
3780 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
3781 }
3782 return Register();
3783}
3784
3785Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3786 switch (VT.SimpleTy) {
3787 case MVT::v8i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(RetVT, Op0);
3788 case MVT::v4i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(RetVT, Op0);
3789 case MVT::v2i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(RetVT, Op0);
3790 default: return Register();
3791 }
3792}
3793
3794// Top-level FastEmit function.
3795
3796Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
3797 switch (Opcode) {
3798 case ARMISD::VADDVs: return fastEmit_ARMISD_VADDVs_r(VT, RetVT, Op0);
3799 case ARMISD::VADDVu: return fastEmit_ARMISD_VADDVu_r(VT, RetVT, Op0);
3800 case ARMISD::VDUP: return fastEmit_ARMISD_VDUP_r(VT, RetVT, Op0);
3801 case ARMISD::VMOVSR: return fastEmit_ARMISD_VMOVSR_r(VT, RetVT, Op0);
3802 case ARMISD::VMOVhr: return fastEmit_ARMISD_VMOVhr_r(VT, RetVT, Op0);
3803 case ARMISD::VMOVrh: return fastEmit_ARMISD_VMOVrh_r(VT, RetVT, Op0);
3804 case ARMISD::VREV16: return fastEmit_ARMISD_VREV16_r(VT, RetVT, Op0);
3805 case ARMISD::VREV32: return fastEmit_ARMISD_VREV32_r(VT, RetVT, Op0);
3806 case ARMISD::VREV64: return fastEmit_ARMISD_VREV64_r(VT, RetVT, Op0);
3807 case ARMISD::WLSSETUP: return fastEmit_ARMISD_WLSSETUP_r(VT, RetVT, Op0);
3808 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
3809 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
3810 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
3811 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
3812 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
3813 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
3814 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
3815 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
3816 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
3817 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
3818 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
3819 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
3820 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
3821 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
3822 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
3823 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
3824 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
3825 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
3826 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
3827 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
3828 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
3829 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
3830 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
3831 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
3832 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
3833 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
3834 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
3835 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
3836 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
3837 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
3838 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
3839 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
3840 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
3841 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
3842 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
3843 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
3844 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
3845 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
3846 default: return Register();
3847 }
3848}
3849
3850// FastEmit functions for ARMISD::QADD16b.
3851
3852Register fastEmit_ARMISD_QADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3853 if (RetVT.SimpleTy != MVT::i32)
3854 return Register();
3855 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3856 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
3857 }
3858 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3859 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3860 }
3861 return Register();
3862}
3863
3864Register fastEmit_ARMISD_QADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3865 switch (VT.SimpleTy) {
3866 case MVT::i32: return fastEmit_ARMISD_QADD16b_MVT_i32_rr(RetVT, Op0, Op1);
3867 default: return Register();
3868 }
3869}
3870
3871// FastEmit functions for ARMISD::QADD8b.
3872
3873Register fastEmit_ARMISD_QADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3874 if (RetVT.SimpleTy != MVT::i32)
3875 return Register();
3876 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3877 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
3878 }
3879 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3880 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3881 }
3882 return Register();
3883}
3884
3885Register fastEmit_ARMISD_QADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3886 switch (VT.SimpleTy) {
3887 case MVT::i32: return fastEmit_ARMISD_QADD8b_MVT_i32_rr(RetVT, Op0, Op1);
3888 default: return Register();
3889 }
3890}
3891
3892// FastEmit functions for ARMISD::QSUB16b.
3893
3894Register fastEmit_ARMISD_QSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3895 if (RetVT.SimpleTy != MVT::i32)
3896 return Register();
3897 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3898 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
3899 }
3900 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3901 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3902 }
3903 return Register();
3904}
3905
3906Register fastEmit_ARMISD_QSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3907 switch (VT.SimpleTy) {
3908 case MVT::i32: return fastEmit_ARMISD_QSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
3909 default: return Register();
3910 }
3911}
3912
3913// FastEmit functions for ARMISD::QSUB8b.
3914
3915Register fastEmit_ARMISD_QSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3916 if (RetVT.SimpleTy != MVT::i32)
3917 return Register();
3918 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3919 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
3920 }
3921 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3922 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3923 }
3924 return Register();
3925}
3926
3927Register fastEmit_ARMISD_QSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3928 switch (VT.SimpleTy) {
3929 case MVT::i32: return fastEmit_ARMISD_QSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
3930 default: return Register();
3931 }
3932}
3933
3934// FastEmit functions for ARMISD::SMULWB.
3935
3936Register fastEmit_ARMISD_SMULWB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3937 if (RetVT.SimpleTy != MVT::i32)
3938 return Register();
3939 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3940 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWB, RC: &ARM::rGPRRegClass, Op0, Op1);
3941 }
3942 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3943 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWB, RC: &ARM::GPRRegClass, Op0, Op1);
3944 }
3945 return Register();
3946}
3947
3948Register fastEmit_ARMISD_SMULWB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3949 switch (VT.SimpleTy) {
3950 case MVT::i32: return fastEmit_ARMISD_SMULWB_MVT_i32_rr(RetVT, Op0, Op1);
3951 default: return Register();
3952 }
3953}
3954
3955// FastEmit functions for ARMISD::SMULWT.
3956
3957Register fastEmit_ARMISD_SMULWT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3958 if (RetVT.SimpleTy != MVT::i32)
3959 return Register();
3960 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3961 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWT, RC: &ARM::rGPRRegClass, Op0, Op1);
3962 }
3963 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3964 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWT, RC: &ARM::GPRRegClass, Op0, Op1);
3965 }
3966 return Register();
3967}
3968
3969Register fastEmit_ARMISD_SMULWT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3970 switch (VT.SimpleTy) {
3971 case MVT::i32: return fastEmit_ARMISD_SMULWT_MVT_i32_rr(RetVT, Op0, Op1);
3972 default: return Register();
3973 }
3974}
3975
3976// FastEmit functions for ARMISD::UQADD16b.
3977
3978Register fastEmit_ARMISD_UQADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3979 if (RetVT.SimpleTy != MVT::i32)
3980 return Register();
3981 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3982 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
3983 }
3984 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3985 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3986 }
3987 return Register();
3988}
3989
3990Register fastEmit_ARMISD_UQADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3991 switch (VT.SimpleTy) {
3992 case MVT::i32: return fastEmit_ARMISD_UQADD16b_MVT_i32_rr(RetVT, Op0, Op1);
3993 default: return Register();
3994 }
3995}
3996
3997// FastEmit functions for ARMISD::UQADD8b.
3998
3999Register fastEmit_ARMISD_UQADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4000 if (RetVT.SimpleTy != MVT::i32)
4001 return Register();
4002 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4003 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
4004 }
4005 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4006 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4007 }
4008 return Register();
4009}
4010
4011Register fastEmit_ARMISD_UQADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4012 switch (VT.SimpleTy) {
4013 case MVT::i32: return fastEmit_ARMISD_UQADD8b_MVT_i32_rr(RetVT, Op0, Op1);
4014 default: return Register();
4015 }
4016}
4017
4018// FastEmit functions for ARMISD::UQSUB16b.
4019
4020Register fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4021 if (RetVT.SimpleTy != MVT::i32)
4022 return Register();
4023 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4024 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
4025 }
4026 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4027 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4028 }
4029 return Register();
4030}
4031
4032Register fastEmit_ARMISD_UQSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4033 switch (VT.SimpleTy) {
4034 case MVT::i32: return fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
4035 default: return Register();
4036 }
4037}
4038
4039// FastEmit functions for ARMISD::UQSUB8b.
4040
4041Register fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4042 if (RetVT.SimpleTy != MVT::i32)
4043 return Register();
4044 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4045 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
4046 }
4047 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4048 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4049 }
4050 return Register();
4051}
4052
4053Register fastEmit_ARMISD_UQSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4054 switch (VT.SimpleTy) {
4055 case MVT::i32: return fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
4056 default: return Register();
4057 }
4058}
4059
4060// FastEmit functions for ARMISD::VMLAVs.
4061
4062Register fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4063 if (RetVT.SimpleTy != MVT::i32)
4064 return Register();
4065 if ((Subtarget->hasMVEIntegerOps())) {
4066 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4067 }
4068 return Register();
4069}
4070
4071Register fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4072 if (RetVT.SimpleTy != MVT::i32)
4073 return Register();
4074 if ((Subtarget->hasMVEIntegerOps())) {
4075 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4076 }
4077 return Register();
4078}
4079
4080Register fastEmit_ARMISD_VMLAVs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4081 switch (VT.SimpleTy) {
4082 case MVT::v16i8: return fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(RetVT, Op0, Op1);
4083 case MVT::v8i16: return fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(RetVT, Op0, Op1);
4084 default: return Register();
4085 }
4086}
4087
4088// FastEmit functions for ARMISD::VMLAVu.
4089
4090Register fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4091 if (RetVT.SimpleTy != MVT::i32)
4092 return Register();
4093 if ((Subtarget->hasMVEIntegerOps())) {
4094 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4095 }
4096 return Register();
4097}
4098
4099Register fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4100 if (RetVT.SimpleTy != MVT::i32)
4101 return Register();
4102 if ((Subtarget->hasMVEIntegerOps())) {
4103 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4104 }
4105 return Register();
4106}
4107
4108Register fastEmit_ARMISD_VMLAVu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4109 switch (VT.SimpleTy) {
4110 case MVT::v16i8: return fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(RetVT, Op0, Op1);
4111 case MVT::v8i16: return fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(RetVT, Op0, Op1);
4112 default: return Register();
4113 }
4114}
4115
4116// FastEmit functions for ARMISD::VMOVDRR.
4117
4118Register fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4119 if (RetVT.SimpleTy != MVT::f64)
4120 return Register();
4121 if ((Subtarget->hasFPRegs())) {
4122 return fastEmitInst_rr(MachineInstOpcode: ARM::VMOVDRR, RC: &ARM::DPRRegClass, Op0, Op1);
4123 }
4124 return Register();
4125}
4126
4127Register fastEmit_ARMISD_VMOVDRR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4128 switch (VT.SimpleTy) {
4129 case MVT::i32: return fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(RetVT, Op0, Op1);
4130 default: return Register();
4131 }
4132}
4133
4134// FastEmit functions for ARMISD::VMULLs.
4135
4136Register fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4137 if (RetVT.SimpleTy != MVT::v8i16)
4138 return Register();
4139 if ((Subtarget->hasNEON())) {
4140 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4141 }
4142 return Register();
4143}
4144
4145Register fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4146 if (RetVT.SimpleTy != MVT::v4i32)
4147 return Register();
4148 if ((Subtarget->hasNEON())) {
4149 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4150 }
4151 return Register();
4152}
4153
4154Register fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4155 if (RetVT.SimpleTy != MVT::v2i64)
4156 return Register();
4157 if ((Subtarget->hasNEON())) {
4158 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4159 }
4160 return Register();
4161}
4162
4163Register fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4164 if (RetVT.SimpleTy != MVT::v2i64)
4165 return Register();
4166 if ((Subtarget->hasMVEIntegerOps())) {
4167 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4168 }
4169 return Register();
4170}
4171
4172Register fastEmit_ARMISD_VMULLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4173 switch (VT.SimpleTy) {
4174 case MVT::v8i8: return fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4175 case MVT::v4i16: return fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4176 case MVT::v2i32: return fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4177 case MVT::v4i32: return fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4178 default: return Register();
4179 }
4180}
4181
4182// FastEmit functions for ARMISD::VMULLu.
4183
4184Register fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4185 if (RetVT.SimpleTy != MVT::v8i16)
4186 return Register();
4187 if ((Subtarget->hasNEON())) {
4188 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4189 }
4190 return Register();
4191}
4192
4193Register fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4194 if (RetVT.SimpleTy != MVT::v4i32)
4195 return Register();
4196 if ((Subtarget->hasNEON())) {
4197 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4198 }
4199 return Register();
4200}
4201
4202Register fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4203 if (RetVT.SimpleTy != MVT::v2i64)
4204 return Register();
4205 if ((Subtarget->hasNEON())) {
4206 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4207 }
4208 return Register();
4209}
4210
4211Register fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4212 if (RetVT.SimpleTy != MVT::v2i64)
4213 return Register();
4214 if ((Subtarget->hasMVEIntegerOps())) {
4215 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4216 }
4217 return Register();
4218}
4219
4220Register fastEmit_ARMISD_VMULLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4221 switch (VT.SimpleTy) {
4222 case MVT::v8i8: return fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4223 case MVT::v4i16: return fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4224 case MVT::v2i32: return fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4225 case MVT::v4i32: return fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4226 default: return Register();
4227 }
4228}
4229
4230// FastEmit functions for ARMISD::VQDMULH.
4231
4232Register fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4233 if (RetVT.SimpleTy != MVT::v16i8)
4234 return Register();
4235 if ((Subtarget->hasMVEIntegerOps())) {
4236 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi8, RC: &ARM::MQPRRegClass, Op0, Op1);
4237 }
4238 return Register();
4239}
4240
4241Register fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4242 if (RetVT.SimpleTy != MVT::v8i16)
4243 return Register();
4244 if ((Subtarget->hasMVEIntegerOps())) {
4245 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi16, RC: &ARM::MQPRRegClass, Op0, Op1);
4246 }
4247 return Register();
4248}
4249
4250Register fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4251 if (RetVT.SimpleTy != MVT::v4i32)
4252 return Register();
4253 if ((Subtarget->hasMVEIntegerOps())) {
4254 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi32, RC: &ARM::MQPRRegClass, Op0, Op1);
4255 }
4256 return Register();
4257}
4258
4259Register fastEmit_ARMISD_VQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4260 switch (VT.SimpleTy) {
4261 case MVT::v16i8: return fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(RetVT, Op0, Op1);
4262 case MVT::v8i16: return fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
4263 case MVT::v4i32: return fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
4264 default: return Register();
4265 }
4266}
4267
4268// FastEmit functions for ARMISD::VSHLs.
4269
4270Register fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4271 if (RetVT.SimpleTy != MVT::v8i8)
4272 return Register();
4273 if ((Subtarget->hasNEON())) {
4274 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4275 }
4276 return Register();
4277}
4278
4279Register fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4280 if (RetVT.SimpleTy != MVT::v16i8)
4281 return Register();
4282 if ((Subtarget->hasMVEIntegerOps())) {
4283 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4284 }
4285 if ((Subtarget->hasNEON())) {
4286 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4287 }
4288 return Register();
4289}
4290
4291Register fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4292 if (RetVT.SimpleTy != MVT::v4i16)
4293 return Register();
4294 if ((Subtarget->hasNEON())) {
4295 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4296 }
4297 return Register();
4298}
4299
4300Register fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4301 if (RetVT.SimpleTy != MVT::v8i16)
4302 return Register();
4303 if ((Subtarget->hasMVEIntegerOps())) {
4304 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4305 }
4306 if ((Subtarget->hasNEON())) {
4307 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4308 }
4309 return Register();
4310}
4311
4312Register fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4313 if (RetVT.SimpleTy != MVT::v2i32)
4314 return Register();
4315 if ((Subtarget->hasNEON())) {
4316 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4317 }
4318 return Register();
4319}
4320
4321Register fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4322 if (RetVT.SimpleTy != MVT::v4i32)
4323 return Register();
4324 if ((Subtarget->hasMVEIntegerOps())) {
4325 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4326 }
4327 if ((Subtarget->hasNEON())) {
4328 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4329 }
4330 return Register();
4331}
4332
4333Register fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4334 if (RetVT.SimpleTy != MVT::v1i64)
4335 return Register();
4336 if ((Subtarget->hasNEON())) {
4337 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4338 }
4339 return Register();
4340}
4341
4342Register fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4343 if (RetVT.SimpleTy != MVT::v2i64)
4344 return Register();
4345 if ((Subtarget->hasNEON())) {
4346 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4347 }
4348 return Register();
4349}
4350
4351Register fastEmit_ARMISD_VSHLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4352 switch (VT.SimpleTy) {
4353 case MVT::v8i8: return fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4354 case MVT::v16i8: return fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(RetVT, Op0, Op1);
4355 case MVT::v4i16: return fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4356 case MVT::v8i16: return fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(RetVT, Op0, Op1);
4357 case MVT::v2i32: return fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4358 case MVT::v4i32: return fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4359 case MVT::v1i64: return fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(RetVT, Op0, Op1);
4360 case MVT::v2i64: return fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(RetVT, Op0, Op1);
4361 default: return Register();
4362 }
4363}
4364
4365// FastEmit functions for ARMISD::VSHLu.
4366
4367Register fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4368 if (RetVT.SimpleTy != MVT::v8i8)
4369 return Register();
4370 if ((Subtarget->hasNEON())) {
4371 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4372 }
4373 return Register();
4374}
4375
4376Register fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4377 if (RetVT.SimpleTy != MVT::v16i8)
4378 return Register();
4379 if ((Subtarget->hasMVEIntegerOps())) {
4380 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4381 }
4382 if ((Subtarget->hasNEON())) {
4383 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4384 }
4385 return Register();
4386}
4387
4388Register fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4389 if (RetVT.SimpleTy != MVT::v4i16)
4390 return Register();
4391 if ((Subtarget->hasNEON())) {
4392 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4393 }
4394 return Register();
4395}
4396
4397Register fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4398 if (RetVT.SimpleTy != MVT::v8i16)
4399 return Register();
4400 if ((Subtarget->hasMVEIntegerOps())) {
4401 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4402 }
4403 if ((Subtarget->hasNEON())) {
4404 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4405 }
4406 return Register();
4407}
4408
4409Register fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4410 if (RetVT.SimpleTy != MVT::v2i32)
4411 return Register();
4412 if ((Subtarget->hasNEON())) {
4413 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4414 }
4415 return Register();
4416}
4417
4418Register fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4419 if (RetVT.SimpleTy != MVT::v4i32)
4420 return Register();
4421 if ((Subtarget->hasMVEIntegerOps())) {
4422 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4423 }
4424 if ((Subtarget->hasNEON())) {
4425 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4426 }
4427 return Register();
4428}
4429
4430Register fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4431 if (RetVT.SimpleTy != MVT::v1i64)
4432 return Register();
4433 if ((Subtarget->hasNEON())) {
4434 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4435 }
4436 return Register();
4437}
4438
4439Register fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4440 if (RetVT.SimpleTy != MVT::v2i64)
4441 return Register();
4442 if ((Subtarget->hasNEON())) {
4443 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4444 }
4445 return Register();
4446}
4447
4448Register fastEmit_ARMISD_VSHLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4449 switch (VT.SimpleTy) {
4450 case MVT::v8i8: return fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4451 case MVT::v16i8: return fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(RetVT, Op0, Op1);
4452 case MVT::v4i16: return fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4453 case MVT::v8i16: return fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(RetVT, Op0, Op1);
4454 case MVT::v2i32: return fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4455 case MVT::v4i32: return fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4456 case MVT::v1i64: return fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(RetVT, Op0, Op1);
4457 case MVT::v2i64: return fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(RetVT, Op0, Op1);
4458 default: return Register();
4459 }
4460}
4461
4462// FastEmit functions for ARMISD::VTBL1.
4463
4464Register fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4465 if (RetVT.SimpleTy != MVT::v8i8)
4466 return Register();
4467 if ((Subtarget->hasNEON())) {
4468 return fastEmitInst_rr(MachineInstOpcode: ARM::VTBL1, RC: &ARM::DPRRegClass, Op0, Op1);
4469 }
4470 return Register();
4471}
4472
4473Register fastEmit_ARMISD_VTBL1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4474 switch (VT.SimpleTy) {
4475 case MVT::v8i8: return fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(RetVT, Op0, Op1);
4476 default: return Register();
4477 }
4478}
4479
4480// FastEmit functions for ARMISD::VTST.
4481
4482Register fastEmit_ARMISD_VTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4483 if (RetVT.SimpleTy != MVT::v8i8)
4484 return Register();
4485 if ((Subtarget->hasNEON())) {
4486 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4487 }
4488 return Register();
4489}
4490
4491Register fastEmit_ARMISD_VTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4492 if (RetVT.SimpleTy != MVT::v16i8)
4493 return Register();
4494 if ((Subtarget->hasNEON())) {
4495 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4496 }
4497 return Register();
4498}
4499
4500Register fastEmit_ARMISD_VTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4501 if (RetVT.SimpleTy != MVT::v4i16)
4502 return Register();
4503 if ((Subtarget->hasNEON())) {
4504 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4505 }
4506 return Register();
4507}
4508
4509Register fastEmit_ARMISD_VTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4510 if (RetVT.SimpleTy != MVT::v8i16)
4511 return Register();
4512 if ((Subtarget->hasNEON())) {
4513 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4514 }
4515 return Register();
4516}
4517
4518Register fastEmit_ARMISD_VTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4519 if (RetVT.SimpleTy != MVT::v2i32)
4520 return Register();
4521 if ((Subtarget->hasNEON())) {
4522 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4523 }
4524 return Register();
4525}
4526
4527Register fastEmit_ARMISD_VTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4528 if (RetVT.SimpleTy != MVT::v4i32)
4529 return Register();
4530 if ((Subtarget->hasNEON())) {
4531 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4532 }
4533 return Register();
4534}
4535
4536Register fastEmit_ARMISD_VTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4537 switch (VT.SimpleTy) {
4538 case MVT::v8i8: return fastEmit_ARMISD_VTST_MVT_v8i8_rr(RetVT, Op0, Op1);
4539 case MVT::v16i8: return fastEmit_ARMISD_VTST_MVT_v16i8_rr(RetVT, Op0, Op1);
4540 case MVT::v4i16: return fastEmit_ARMISD_VTST_MVT_v4i16_rr(RetVT, Op0, Op1);
4541 case MVT::v8i16: return fastEmit_ARMISD_VTST_MVT_v8i16_rr(RetVT, Op0, Op1);
4542 case MVT::v2i32: return fastEmit_ARMISD_VTST_MVT_v2i32_rr(RetVT, Op0, Op1);
4543 case MVT::v4i32: return fastEmit_ARMISD_VTST_MVT_v4i32_rr(RetVT, Op0, Op1);
4544 default: return Register();
4545 }
4546}
4547
4548// FastEmit functions for ISD::ABDS.
4549
4550Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4551 if (RetVT.SimpleTy != MVT::v8i8)
4552 return Register();
4553 if ((Subtarget->hasNEON())) {
4554 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4555 }
4556 return Register();
4557}
4558
4559Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4560 if (RetVT.SimpleTy != MVT::v16i8)
4561 return Register();
4562 if ((Subtarget->hasMVEIntegerOps())) {
4563 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4564 }
4565 if ((Subtarget->hasNEON())) {
4566 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4567 }
4568 return Register();
4569}
4570
4571Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4572 if (RetVT.SimpleTy != MVT::v4i16)
4573 return Register();
4574 if ((Subtarget->hasNEON())) {
4575 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4576 }
4577 return Register();
4578}
4579
4580Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4581 if (RetVT.SimpleTy != MVT::v8i16)
4582 return Register();
4583 if ((Subtarget->hasMVEIntegerOps())) {
4584 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4585 }
4586 if ((Subtarget->hasNEON())) {
4587 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4588 }
4589 return Register();
4590}
4591
4592Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4593 if (RetVT.SimpleTy != MVT::v2i32)
4594 return Register();
4595 if ((Subtarget->hasNEON())) {
4596 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4597 }
4598 return Register();
4599}
4600
4601Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4602 if (RetVT.SimpleTy != MVT::v4i32)
4603 return Register();
4604 if ((Subtarget->hasMVEIntegerOps())) {
4605 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4606 }
4607 if ((Subtarget->hasNEON())) {
4608 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4609 }
4610 return Register();
4611}
4612
4613Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4614 switch (VT.SimpleTy) {
4615 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
4616 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
4617 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
4618 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
4619 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
4620 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
4621 default: return Register();
4622 }
4623}
4624
4625// FastEmit functions for ISD::ABDU.
4626
4627Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4628 if (RetVT.SimpleTy != MVT::v8i8)
4629 return Register();
4630 if ((Subtarget->hasNEON())) {
4631 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4632 }
4633 return Register();
4634}
4635
4636Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4637 if (RetVT.SimpleTy != MVT::v16i8)
4638 return Register();
4639 if ((Subtarget->hasMVEIntegerOps())) {
4640 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4641 }
4642 if ((Subtarget->hasNEON())) {
4643 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4644 }
4645 return Register();
4646}
4647
4648Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4649 if (RetVT.SimpleTy != MVT::v4i16)
4650 return Register();
4651 if ((Subtarget->hasNEON())) {
4652 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4653 }
4654 return Register();
4655}
4656
4657Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4658 if (RetVT.SimpleTy != MVT::v8i16)
4659 return Register();
4660 if ((Subtarget->hasMVEIntegerOps())) {
4661 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4662 }
4663 if ((Subtarget->hasNEON())) {
4664 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4665 }
4666 return Register();
4667}
4668
4669Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4670 if (RetVT.SimpleTy != MVT::v2i32)
4671 return Register();
4672 if ((Subtarget->hasNEON())) {
4673 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4674 }
4675 return Register();
4676}
4677
4678Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4679 if (RetVT.SimpleTy != MVT::v4i32)
4680 return Register();
4681 if ((Subtarget->hasMVEIntegerOps())) {
4682 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4683 }
4684 if ((Subtarget->hasNEON())) {
4685 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4686 }
4687 return Register();
4688}
4689
4690Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4691 switch (VT.SimpleTy) {
4692 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
4693 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
4694 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
4695 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
4696 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
4697 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
4698 default: return Register();
4699 }
4700}
4701
4702// FastEmit functions for ISD::ADD.
4703
4704Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4705 if (RetVT.SimpleTy != MVT::i32)
4706 return Register();
4707 if ((Subtarget->isThumb2())) {
4708 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ADDrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4709 }
4710 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4711 return fastEmitInst_rr(MachineInstOpcode: ARM::tADDrr, RC: &ARM::tGPRRegClass, Op0, Op1);
4712 }
4713 if ((!Subtarget->isThumb())) {
4714 return fastEmitInst_rr(MachineInstOpcode: ARM::ADDrr, RC: &ARM::GPRRegClass, Op0, Op1);
4715 }
4716 return Register();
4717}
4718
4719Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4720 if (RetVT.SimpleTy != MVT::v8i8)
4721 return Register();
4722 if ((Subtarget->hasNEON())) {
4723 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4724 }
4725 return Register();
4726}
4727
4728Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4729 if (RetVT.SimpleTy != MVT::v16i8)
4730 return Register();
4731 if ((Subtarget->hasMVEIntegerOps())) {
4732 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi8, RC: &ARM::MQPRRegClass, Op0, Op1);
4733 }
4734 if ((Subtarget->hasNEON())) {
4735 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4736 }
4737 return Register();
4738}
4739
4740Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4741 if (RetVT.SimpleTy != MVT::v4i16)
4742 return Register();
4743 if ((Subtarget->hasNEON())) {
4744 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4745 }
4746 return Register();
4747}
4748
4749Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4750 if (RetVT.SimpleTy != MVT::v8i16)
4751 return Register();
4752 if ((Subtarget->hasMVEIntegerOps())) {
4753 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi16, RC: &ARM::MQPRRegClass, Op0, Op1);
4754 }
4755 if ((Subtarget->hasNEON())) {
4756 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4757 }
4758 return Register();
4759}
4760
4761Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4762 if (RetVT.SimpleTy != MVT::v2i32)
4763 return Register();
4764 if ((Subtarget->hasNEON())) {
4765 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4766 }
4767 return Register();
4768}
4769
4770Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4771 if (RetVT.SimpleTy != MVT::v4i32)
4772 return Register();
4773 if ((Subtarget->hasMVEIntegerOps())) {
4774 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi32, RC: &ARM::MQPRRegClass, Op0, Op1);
4775 }
4776 if ((Subtarget->hasNEON())) {
4777 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4778 }
4779 return Register();
4780}
4781
4782Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4783 if (RetVT.SimpleTy != MVT::v1i64)
4784 return Register();
4785 if ((Subtarget->hasNEON())) {
4786 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4787 }
4788 return Register();
4789}
4790
4791Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4792 if (RetVT.SimpleTy != MVT::v2i64)
4793 return Register();
4794 if ((Subtarget->hasNEON())) {
4795 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4796 }
4797 return Register();
4798}
4799
4800Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4801 switch (VT.SimpleTy) {
4802 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
4803 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
4804 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
4805 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
4806 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
4807 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
4808 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
4809 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
4810 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
4811 default: return Register();
4812 }
4813}
4814
4815// FastEmit functions for ISD::AND.
4816
4817Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4818 if (RetVT.SimpleTy != MVT::i32)
4819 return Register();
4820 if ((Subtarget->isThumb2())) {
4821 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ANDrr, RC: &ARM::rGPRRegClass, Op0, Op1);
4822 }
4823 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4824 return fastEmitInst_rr(MachineInstOpcode: ARM::tAND, RC: &ARM::tGPRRegClass, Op0, Op1);
4825 }
4826 if ((!Subtarget->isThumb())) {
4827 return fastEmitInst_rr(MachineInstOpcode: ARM::ANDrr, RC: &ARM::GPRRegClass, Op0, Op1);
4828 }
4829 return Register();
4830}
4831
4832Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4833 if (RetVT.SimpleTy != MVT::v8i8)
4834 return Register();
4835 if ((Subtarget->hasNEON())) {
4836 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4837 }
4838 return Register();
4839}
4840
4841Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4842 if (RetVT.SimpleTy != MVT::v16i8)
4843 return Register();
4844 if ((Subtarget->hasMVEIntegerOps())) {
4845 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4846 }
4847 if ((Subtarget->hasNEON())) {
4848 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4849 }
4850 return Register();
4851}
4852
4853Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4854 if (RetVT.SimpleTy != MVT::v4i16)
4855 return Register();
4856 if ((Subtarget->hasNEON())) {
4857 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4858 }
4859 return Register();
4860}
4861
4862Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4863 if (RetVT.SimpleTy != MVT::v8i16)
4864 return Register();
4865 if ((Subtarget->hasMVEIntegerOps())) {
4866 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4867 }
4868 if ((Subtarget->hasNEON())) {
4869 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4870 }
4871 return Register();
4872}
4873
4874Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4875 if (RetVT.SimpleTy != MVT::v2i32)
4876 return Register();
4877 if ((Subtarget->hasNEON())) {
4878 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4879 }
4880 return Register();
4881}
4882
4883Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4884 if (RetVT.SimpleTy != MVT::v4i32)
4885 return Register();
4886 if ((Subtarget->hasMVEIntegerOps())) {
4887 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4888 }
4889 if ((Subtarget->hasNEON())) {
4890 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4891 }
4892 return Register();
4893}
4894
4895Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4896 if (RetVT.SimpleTy != MVT::v1i64)
4897 return Register();
4898 if ((Subtarget->hasNEON())) {
4899 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4900 }
4901 return Register();
4902}
4903
4904Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4905 if (RetVT.SimpleTy != MVT::v2i64)
4906 return Register();
4907 if ((Subtarget->hasMVEIntegerOps())) {
4908 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4909 }
4910 if ((Subtarget->hasNEON())) {
4911 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4912 }
4913 return Register();
4914}
4915
4916Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4917 switch (VT.SimpleTy) {
4918 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
4919 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
4920 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
4921 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
4922 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
4923 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
4924 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
4925 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
4926 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
4927 default: return Register();
4928 }
4929}
4930
4931// FastEmit functions for ISD::AVGCEILS.
4932
4933Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4934 if (RetVT.SimpleTy != MVT::v16i8)
4935 return Register();
4936 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4937}
4938
4939Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4940 if (RetVT.SimpleTy != MVT::v8i16)
4941 return Register();
4942 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4943}
4944
4945Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4946 if (RetVT.SimpleTy != MVT::v4i32)
4947 return Register();
4948 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4949}
4950
4951Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4952 switch (VT.SimpleTy) {
4953 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
4954 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
4955 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
4956 default: return Register();
4957 }
4958}
4959
4960// FastEmit functions for ISD::AVGCEILU.
4961
4962Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4963 if (RetVT.SimpleTy != MVT::v16i8)
4964 return Register();
4965 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4966}
4967
4968Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4969 if (RetVT.SimpleTy != MVT::v8i16)
4970 return Register();
4971 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4972}
4973
4974Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4975 if (RetVT.SimpleTy != MVT::v4i32)
4976 return Register();
4977 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4978}
4979
4980Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4981 switch (VT.SimpleTy) {
4982 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
4983 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
4984 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
4985 default: return Register();
4986 }
4987}
4988
4989// FastEmit functions for ISD::AVGFLOORS.
4990
4991Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4992 if (RetVT.SimpleTy != MVT::v16i8)
4993 return Register();
4994 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4995}
4996
4997Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4998 if (RetVT.SimpleTy != MVT::v8i16)
4999 return Register();
5000 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5001}
5002
5003Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5004 if (RetVT.SimpleTy != MVT::v4i32)
5005 return Register();
5006 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5007}
5008
5009Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5010 switch (VT.SimpleTy) {
5011 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
5012 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
5013 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
5014 default: return Register();
5015 }
5016}
5017
5018// FastEmit functions for ISD::AVGFLOORU.
5019
5020Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5021 if (RetVT.SimpleTy != MVT::v16i8)
5022 return Register();
5023 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5024}
5025
5026Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5027 if (RetVT.SimpleTy != MVT::v8i16)
5028 return Register();
5029 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5030}
5031
5032Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5033 if (RetVT.SimpleTy != MVT::v4i32)
5034 return Register();
5035 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5036}
5037
5038Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5039 switch (VT.SimpleTy) {
5040 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
5041 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
5042 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
5043 default: return Register();
5044 }
5045}
5046
5047// FastEmit functions for ISD::FADD.
5048
5049Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5050 if (RetVT.SimpleTy != MVT::f16)
5051 return Register();
5052 if ((Subtarget->hasFullFP16())) {
5053 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
5054 }
5055 return Register();
5056}
5057
5058Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5059 if (RetVT.SimpleTy != MVT::f32)
5060 return Register();
5061 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5062 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
5063 }
5064 return Register();
5065}
5066
5067Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5068 if (RetVT.SimpleTy != MVT::f64)
5069 return Register();
5070 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5071 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
5072 }
5073 return Register();
5074}
5075
5076Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5077 if (RetVT.SimpleTy != MVT::v4f16)
5078 return Register();
5079 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5080 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhd, RC: &ARM::DPRRegClass, Op0, Op1);
5081 }
5082 return Register();
5083}
5084
5085Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5086 if (RetVT.SimpleTy != MVT::v8f16)
5087 return Register();
5088 if ((Subtarget->hasMVEFloatOps())) {
5089 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5090 }
5091 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5092 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhq, RC: &ARM::QPRRegClass, Op0, Op1);
5093 }
5094 return Register();
5095}
5096
5097Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5098 if (RetVT.SimpleTy != MVT::v2f32)
5099 return Register();
5100 if ((Subtarget->hasNEON())) {
5101 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfd, RC: &ARM::DPRRegClass, Op0, Op1);
5102 }
5103 return Register();
5104}
5105
5106Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5107 if (RetVT.SimpleTy != MVT::v4f32)
5108 return Register();
5109 if ((Subtarget->hasMVEFloatOps())) {
5110 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5111 }
5112 if ((Subtarget->hasNEON())) {
5113 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfq, RC: &ARM::QPRRegClass, Op0, Op1);
5114 }
5115 return Register();
5116}
5117
5118Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5119 switch (VT.SimpleTy) {
5120 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
5121 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
5122 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
5123 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
5124 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
5125 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
5126 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
5127 default: return Register();
5128 }
5129}
5130
5131// FastEmit functions for ISD::FDIV.
5132
5133Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5134 if (RetVT.SimpleTy != MVT::f16)
5135 return Register();
5136 if ((Subtarget->hasFullFP16())) {
5137 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
5138 }
5139 return Register();
5140}
5141
5142Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5143 if (RetVT.SimpleTy != MVT::f32)
5144 return Register();
5145 if ((Subtarget->hasVFP2Base())) {
5146 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
5147 }
5148 return Register();
5149}
5150
5151Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5152 if (RetVT.SimpleTy != MVT::f64)
5153 return Register();
5154 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5155 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
5156 }
5157 return Register();
5158}
5159
5160Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5161 switch (VT.SimpleTy) {
5162 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
5163 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
5164 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
5165 default: return Register();
5166 }
5167}
5168
5169// FastEmit functions for ISD::FMAXIMUM.
5170
5171Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5172 if (RetVT.SimpleTy != MVT::v4f16)
5173 return Register();
5174 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5175 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhd, RC: &ARM::DPRRegClass, Op0, Op1);
5176 }
5177 return Register();
5178}
5179
5180Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5181 if (RetVT.SimpleTy != MVT::v8f16)
5182 return Register();
5183 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5184 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhq, RC: &ARM::QPRRegClass, Op0, Op1);
5185 }
5186 return Register();
5187}
5188
5189Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5190 if (RetVT.SimpleTy != MVT::v2f32)
5191 return Register();
5192 if ((Subtarget->hasNEON())) {
5193 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfd, RC: &ARM::DPRRegClass, Op0, Op1);
5194 }
5195 return Register();
5196}
5197
5198Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5199 if (RetVT.SimpleTy != MVT::v4f32)
5200 return Register();
5201 if ((Subtarget->hasNEON())) {
5202 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfq, RC: &ARM::QPRRegClass, Op0, Op1);
5203 }
5204 return Register();
5205}
5206
5207Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5208 switch (VT.SimpleTy) {
5209 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5210 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5211 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5212 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5213 default: return Register();
5214 }
5215}
5216
5217// FastEmit functions for ISD::FMAXNUM.
5218
5219Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5220 if (RetVT.SimpleTy != MVT::f16)
5221 return Register();
5222 if ((Subtarget->hasFullFP16())) {
5223 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5224 }
5225 return Register();
5226}
5227
5228Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5229 if (RetVT.SimpleTy != MVT::f32)
5230 return Register();
5231 if ((Subtarget->hasFPARMv8Base())) {
5232 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5233 }
5234 return Register();
5235}
5236
5237Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5238 if (RetVT.SimpleTy != MVT::f64)
5239 return Register();
5240 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5241 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5242 }
5243 return Register();
5244}
5245
5246Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5247 if (RetVT.SimpleTy != MVT::v4f16)
5248 return Register();
5249 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5250 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5251 }
5252 return Register();
5253}
5254
5255Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5256 if (RetVT.SimpleTy != MVT::v8f16)
5257 return Register();
5258 if ((Subtarget->hasMVEFloatOps())) {
5259 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5260 }
5261 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5262 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5263 }
5264 return Register();
5265}
5266
5267Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5268 if (RetVT.SimpleTy != MVT::v2f32)
5269 return Register();
5270 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5271 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5272 }
5273 return Register();
5274}
5275
5276Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5277 if (RetVT.SimpleTy != MVT::v4f32)
5278 return Register();
5279 if ((Subtarget->hasMVEFloatOps())) {
5280 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5281 }
5282 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5283 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5284 }
5285 return Register();
5286}
5287
5288Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5289 switch (VT.SimpleTy) {
5290 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
5291 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
5292 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
5293 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5294 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5295 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5296 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5297 default: return Register();
5298 }
5299}
5300
5301// FastEmit functions for ISD::FMINIMUM.
5302
5303Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5304 if (RetVT.SimpleTy != MVT::v4f16)
5305 return Register();
5306 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5307 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhd, RC: &ARM::DPRRegClass, Op0, Op1);
5308 }
5309 return Register();
5310}
5311
5312Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5313 if (RetVT.SimpleTy != MVT::v8f16)
5314 return Register();
5315 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5316 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhq, RC: &ARM::QPRRegClass, Op0, Op1);
5317 }
5318 return Register();
5319}
5320
5321Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5322 if (RetVT.SimpleTy != MVT::v2f32)
5323 return Register();
5324 if ((Subtarget->hasNEON())) {
5325 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfd, RC: &ARM::DPRRegClass, Op0, Op1);
5326 }
5327 return Register();
5328}
5329
5330Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5331 if (RetVT.SimpleTy != MVT::v4f32)
5332 return Register();
5333 if ((Subtarget->hasNEON())) {
5334 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfq, RC: &ARM::QPRRegClass, Op0, Op1);
5335 }
5336 return Register();
5337}
5338
5339Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5340 switch (VT.SimpleTy) {
5341 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5342 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5343 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5344 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5345 default: return Register();
5346 }
5347}
5348
5349// FastEmit functions for ISD::FMINNUM.
5350
5351Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5352 if (RetVT.SimpleTy != MVT::f16)
5353 return Register();
5354 if ((Subtarget->hasFullFP16())) {
5355 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5356 }
5357 return Register();
5358}
5359
5360Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5361 if (RetVT.SimpleTy != MVT::f32)
5362 return Register();
5363 if ((Subtarget->hasFPARMv8Base())) {
5364 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5365 }
5366 return Register();
5367}
5368
5369Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5370 if (RetVT.SimpleTy != MVT::f64)
5371 return Register();
5372 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5373 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5374 }
5375 return Register();
5376}
5377
5378Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5379 if (RetVT.SimpleTy != MVT::v4f16)
5380 return Register();
5381 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5382 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5383 }
5384 return Register();
5385}
5386
5387Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5388 if (RetVT.SimpleTy != MVT::v8f16)
5389 return Register();
5390 if ((Subtarget->hasMVEFloatOps())) {
5391 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5392 }
5393 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5394 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5395 }
5396 return Register();
5397}
5398
5399Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5400 if (RetVT.SimpleTy != MVT::v2f32)
5401 return Register();
5402 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5403 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5404 }
5405 return Register();
5406}
5407
5408Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5409 if (RetVT.SimpleTy != MVT::v4f32)
5410 return Register();
5411 if ((Subtarget->hasMVEFloatOps())) {
5412 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5413 }
5414 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5415 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5416 }
5417 return Register();
5418}
5419
5420Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5421 switch (VT.SimpleTy) {
5422 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
5423 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
5424 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
5425 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5426 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5427 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5428 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5429 default: return Register();
5430 }
5431}
5432
5433// FastEmit functions for ISD::FMUL.
5434
5435Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5436 if (RetVT.SimpleTy != MVT::f16)
5437 return Register();
5438 if ((Subtarget->hasFullFP16())) {
5439 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
5440 }
5441 return Register();
5442}
5443
5444Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5445 if (RetVT.SimpleTy != MVT::f32)
5446 return Register();
5447 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5448 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
5449 }
5450 return Register();
5451}
5452
5453Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5454 if (RetVT.SimpleTy != MVT::f64)
5455 return Register();
5456 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5457 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
5458 }
5459 return Register();
5460}
5461
5462Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5463 if (RetVT.SimpleTy != MVT::v4f16)
5464 return Register();
5465 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5466 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhd, RC: &ARM::DPRRegClass, Op0, Op1);
5467 }
5468 return Register();
5469}
5470
5471Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5472 if (RetVT.SimpleTy != MVT::v8f16)
5473 return Register();
5474 if ((Subtarget->hasMVEFloatOps())) {
5475 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5476 }
5477 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5478 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhq, RC: &ARM::QPRRegClass, Op0, Op1);
5479 }
5480 return Register();
5481}
5482
5483Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5484 if (RetVT.SimpleTy != MVT::v2f32)
5485 return Register();
5486 if ((Subtarget->hasNEON())) {
5487 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfd, RC: &ARM::DPRRegClass, Op0, Op1);
5488 }
5489 return Register();
5490}
5491
5492Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5493 if (RetVT.SimpleTy != MVT::v4f32)
5494 return Register();
5495 if ((Subtarget->hasMVEFloatOps())) {
5496 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5497 }
5498 if ((Subtarget->hasNEON())) {
5499 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfq, RC: &ARM::QPRRegClass, Op0, Op1);
5500 }
5501 return Register();
5502}
5503
5504Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5505 switch (VT.SimpleTy) {
5506 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
5507 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
5508 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
5509 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
5510 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
5511 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
5512 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
5513 default: return Register();
5514 }
5515}
5516
5517// FastEmit functions for ISD::FSUB.
5518
5519Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5520 if (RetVT.SimpleTy != MVT::f16)
5521 return Register();
5522 if ((Subtarget->hasFullFP16())) {
5523 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
5524 }
5525 return Register();
5526}
5527
5528Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5529 if (RetVT.SimpleTy != MVT::f32)
5530 return Register();
5531 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5532 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
5533 }
5534 return Register();
5535}
5536
5537Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5538 if (RetVT.SimpleTy != MVT::f64)
5539 return Register();
5540 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5541 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
5542 }
5543 return Register();
5544}
5545
5546Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5547 if (RetVT.SimpleTy != MVT::v4f16)
5548 return Register();
5549 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5550 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhd, RC: &ARM::DPRRegClass, Op0, Op1);
5551 }
5552 return Register();
5553}
5554
5555Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5556 if (RetVT.SimpleTy != MVT::v8f16)
5557 return Register();
5558 if ((Subtarget->hasMVEFloatOps())) {
5559 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5560 }
5561 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5562 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhq, RC: &ARM::QPRRegClass, Op0, Op1);
5563 }
5564 return Register();
5565}
5566
5567Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5568 if (RetVT.SimpleTy != MVT::v2f32)
5569 return Register();
5570 if ((Subtarget->hasNEON())) {
5571 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfd, RC: &ARM::DPRRegClass, Op0, Op1);
5572 }
5573 return Register();
5574}
5575
5576Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5577 if (RetVT.SimpleTy != MVT::v4f32)
5578 return Register();
5579 if ((Subtarget->hasMVEFloatOps())) {
5580 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5581 }
5582 if ((Subtarget->hasNEON())) {
5583 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfq, RC: &ARM::QPRRegClass, Op0, Op1);
5584 }
5585 return Register();
5586}
5587
5588Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5589 switch (VT.SimpleTy) {
5590 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
5591 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
5592 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
5593 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
5594 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
5595 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
5596 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
5597 default: return Register();
5598 }
5599}
5600
5601// FastEmit functions for ISD::MUL.
5602
5603Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5604 if (RetVT.SimpleTy != MVT::i32)
5605 return Register();
5606 if ((Subtarget->isThumb2())) {
5607 return fastEmitInst_rr(MachineInstOpcode: ARM::t2MUL, RC: &ARM::rGPRRegClass, Op0, Op1);
5608 }
5609 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5610 return fastEmitInst_rr(MachineInstOpcode: ARM::tMUL, RC: &ARM::tGPRRegClass, Op0, Op1);
5611 }
5612 if ((!Subtarget->isThumb()) && (!Subtarget->hasV6Ops()) && (Subtarget->useMulOps())) {
5613 return fastEmitInst_rr(MachineInstOpcode: ARM::MULv5, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5614 }
5615 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
5616 return fastEmitInst_rr(MachineInstOpcode: ARM::MUL, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5617 }
5618 return Register();
5619}
5620
5621Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5622 if (RetVT.SimpleTy != MVT::v8i8)
5623 return Register();
5624 if ((Subtarget->hasNEON())) {
5625 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5626 }
5627 return Register();
5628}
5629
5630Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5631 if (RetVT.SimpleTy != MVT::v16i8)
5632 return Register();
5633 if ((Subtarget->hasMVEIntegerOps())) {
5634 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi8, RC: &ARM::MQPRRegClass, Op0, Op1);
5635 }
5636 if ((Subtarget->hasNEON())) {
5637 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5638 }
5639 return Register();
5640}
5641
5642Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5643 if (RetVT.SimpleTy != MVT::v4i16)
5644 return Register();
5645 if ((Subtarget->hasNEON())) {
5646 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5647 }
5648 return Register();
5649}
5650
5651Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5652 if (RetVT.SimpleTy != MVT::v8i16)
5653 return Register();
5654 if ((Subtarget->hasMVEIntegerOps())) {
5655 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi16, RC: &ARM::MQPRRegClass, Op0, Op1);
5656 }
5657 if ((Subtarget->hasNEON())) {
5658 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5659 }
5660 return Register();
5661}
5662
5663Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5664 if (RetVT.SimpleTy != MVT::v2i32)
5665 return Register();
5666 if ((Subtarget->hasNEON())) {
5667 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5668 }
5669 return Register();
5670}
5671
5672Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5673 if (RetVT.SimpleTy != MVT::v4i32)
5674 return Register();
5675 if ((Subtarget->hasMVEIntegerOps())) {
5676 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi32, RC: &ARM::MQPRRegClass, Op0, Op1);
5677 }
5678 if ((Subtarget->hasNEON())) {
5679 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5680 }
5681 return Register();
5682}
5683
5684Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5685 switch (VT.SimpleTy) {
5686 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
5687 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
5688 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
5689 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
5690 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
5691 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
5692 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
5693 default: return Register();
5694 }
5695}
5696
5697// FastEmit functions for ISD::MULHS.
5698
5699Register fastEmit_ISD_MULHS_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5700 if (RetVT.SimpleTy != MVT::i32)
5701 return Register();
5702 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5703 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMMUL, RC: &ARM::rGPRRegClass, Op0, Op1);
5704 }
5705 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
5706 return fastEmitInst_rr(MachineInstOpcode: ARM::SMMUL, RC: &ARM::GPRRegClass, Op0, Op1);
5707 }
5708 return Register();
5709}
5710
5711Register fastEmit_ISD_MULHS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5712 if (RetVT.SimpleTy != MVT::v16i8)
5713 return Register();
5714 if ((Subtarget->hasMVEIntegerOps())) {
5715 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5716 }
5717 return Register();
5718}
5719
5720Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5721 if (RetVT.SimpleTy != MVT::v8i16)
5722 return Register();
5723 if ((Subtarget->hasMVEIntegerOps())) {
5724 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5725 }
5726 return Register();
5727}
5728
5729Register fastEmit_ISD_MULHS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5730 if (RetVT.SimpleTy != MVT::v4i32)
5731 return Register();
5732 if ((Subtarget->hasMVEIntegerOps())) {
5733 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5734 }
5735 return Register();
5736}
5737
5738Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5739 switch (VT.SimpleTy) {
5740 case MVT::i32: return fastEmit_ISD_MULHS_MVT_i32_rr(RetVT, Op0, Op1);
5741 case MVT::v16i8: return fastEmit_ISD_MULHS_MVT_v16i8_rr(RetVT, Op0, Op1);
5742 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
5743 case MVT::v4i32: return fastEmit_ISD_MULHS_MVT_v4i32_rr(RetVT, Op0, Op1);
5744 default: return Register();
5745 }
5746}
5747
5748// FastEmit functions for ISD::MULHU.
5749
5750Register fastEmit_ISD_MULHU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5751 if (RetVT.SimpleTy != MVT::v16i8)
5752 return Register();
5753 if ((Subtarget->hasMVEIntegerOps())) {
5754 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5755 }
5756 return Register();
5757}
5758
5759Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5760 if (RetVT.SimpleTy != MVT::v8i16)
5761 return Register();
5762 if ((Subtarget->hasMVEIntegerOps())) {
5763 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5764 }
5765 return Register();
5766}
5767
5768Register fastEmit_ISD_MULHU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5769 if (RetVT.SimpleTy != MVT::v4i32)
5770 return Register();
5771 if ((Subtarget->hasMVEIntegerOps())) {
5772 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5773 }
5774 return Register();
5775}
5776
5777Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5778 switch (VT.SimpleTy) {
5779 case MVT::v16i8: return fastEmit_ISD_MULHU_MVT_v16i8_rr(RetVT, Op0, Op1);
5780 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
5781 case MVT::v4i32: return fastEmit_ISD_MULHU_MVT_v4i32_rr(RetVT, Op0, Op1);
5782 default: return Register();
5783 }
5784}
5785
5786// FastEmit functions for ISD::OR.
5787
5788Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5789 if (RetVT.SimpleTy != MVT::i32)
5790 return Register();
5791 if ((Subtarget->isThumb2())) {
5792 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ORRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5793 }
5794 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5795 return fastEmitInst_rr(MachineInstOpcode: ARM::tORR, RC: &ARM::tGPRRegClass, Op0, Op1);
5796 }
5797 if ((!Subtarget->isThumb())) {
5798 return fastEmitInst_rr(MachineInstOpcode: ARM::ORRrr, RC: &ARM::GPRRegClass, Op0, Op1);
5799 }
5800 return Register();
5801}
5802
5803Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5804 if (RetVT.SimpleTy != MVT::v8i8)
5805 return Register();
5806 if ((Subtarget->hasNEON())) {
5807 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5808 }
5809 return Register();
5810}
5811
5812Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5813 if (RetVT.SimpleTy != MVT::v16i8)
5814 return Register();
5815 if ((Subtarget->hasMVEIntegerOps())) {
5816 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5817 }
5818 if ((Subtarget->hasNEON())) {
5819 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5820 }
5821 return Register();
5822}
5823
5824Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5825 if (RetVT.SimpleTy != MVT::v4i16)
5826 return Register();
5827 if ((Subtarget->hasNEON())) {
5828 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5829 }
5830 return Register();
5831}
5832
5833Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5834 if (RetVT.SimpleTy != MVT::v8i16)
5835 return Register();
5836 if ((Subtarget->hasMVEIntegerOps())) {
5837 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5838 }
5839 if ((Subtarget->hasNEON())) {
5840 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5841 }
5842 return Register();
5843}
5844
5845Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5846 if (RetVT.SimpleTy != MVT::v2i32)
5847 return Register();
5848 if ((Subtarget->hasNEON())) {
5849 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5850 }
5851 return Register();
5852}
5853
5854Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5855 if (RetVT.SimpleTy != MVT::v4i32)
5856 return Register();
5857 if ((Subtarget->hasMVEIntegerOps())) {
5858 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5859 }
5860 if ((Subtarget->hasNEON())) {
5861 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5862 }
5863 return Register();
5864}
5865
5866Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
5867 if (RetVT.SimpleTy != MVT::v1i64)
5868 return Register();
5869 if ((Subtarget->hasNEON())) {
5870 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5871 }
5872 return Register();
5873}
5874
5875Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
5876 if (RetVT.SimpleTy != MVT::v2i64)
5877 return Register();
5878 if ((Subtarget->hasMVEIntegerOps())) {
5879 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5880 }
5881 if ((Subtarget->hasNEON())) {
5882 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5883 }
5884 return Register();
5885}
5886
5887Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5888 switch (VT.SimpleTy) {
5889 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
5890 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
5891 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
5892 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
5893 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
5894 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
5895 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
5896 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
5897 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
5898 default: return Register();
5899 }
5900}
5901
5902// FastEmit functions for ISD::ROTR.
5903
5904Register fastEmit_ISD_ROTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5905 if (RetVT.SimpleTy != MVT::i32)
5906 return Register();
5907 if ((Subtarget->isThumb2())) {
5908 return fastEmitInst_rr(MachineInstOpcode: ARM::t2RORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5909 }
5910 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5911 return fastEmitInst_rr(MachineInstOpcode: ARM::tROR, RC: &ARM::tGPRRegClass, Op0, Op1);
5912 }
5913 return Register();
5914}
5915
5916Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5917 switch (VT.SimpleTy) {
5918 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_rr(RetVT, Op0, Op1);
5919 default: return Register();
5920 }
5921}
5922
5923// FastEmit functions for ISD::SADDSAT.
5924
5925Register fastEmit_ISD_SADDSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5926 if (RetVT.SimpleTy != MVT::i32)
5927 return Register();
5928 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5929 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD, RC: &ARM::rGPRRegClass, Op0, Op1);
5930 }
5931 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
5932 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5933 }
5934 return Register();
5935}
5936
5937Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5938 if (RetVT.SimpleTy != MVT::v8i8)
5939 return Register();
5940 if ((Subtarget->hasNEON())) {
5941 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5942 }
5943 return Register();
5944}
5945
5946Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5947 if (RetVT.SimpleTy != MVT::v16i8)
5948 return Register();
5949 if ((Subtarget->hasMVEIntegerOps())) {
5950 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5951 }
5952 if ((Subtarget->hasNEON())) {
5953 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5954 }
5955 return Register();
5956}
5957
5958Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5959 if (RetVT.SimpleTy != MVT::v4i16)
5960 return Register();
5961 if ((Subtarget->hasNEON())) {
5962 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5963 }
5964 return Register();
5965}
5966
5967Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5968 if (RetVT.SimpleTy != MVT::v8i16)
5969 return Register();
5970 if ((Subtarget->hasMVEIntegerOps())) {
5971 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5972 }
5973 if ((Subtarget->hasNEON())) {
5974 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5975 }
5976 return Register();
5977}
5978
5979Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5980 if (RetVT.SimpleTy != MVT::v2i32)
5981 return Register();
5982 if ((Subtarget->hasNEON())) {
5983 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5984 }
5985 return Register();
5986}
5987
5988Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5989 if (RetVT.SimpleTy != MVT::v4i32)
5990 return Register();
5991 if ((Subtarget->hasMVEIntegerOps())) {
5992 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5993 }
5994 if ((Subtarget->hasNEON())) {
5995 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5996 }
5997 return Register();
5998}
5999
6000Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6001 if (RetVT.SimpleTy != MVT::v1i64)
6002 return Register();
6003 if ((Subtarget->hasNEON())) {
6004 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6005 }
6006 return Register();
6007}
6008
6009Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6010 if (RetVT.SimpleTy != MVT::v2i64)
6011 return Register();
6012 if ((Subtarget->hasNEON())) {
6013 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6014 }
6015 return Register();
6016}
6017
6018Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6019 switch (VT.SimpleTy) {
6020 case MVT::i32: return fastEmit_ISD_SADDSAT_MVT_i32_rr(RetVT, Op0, Op1);
6021 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6022 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6023 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6024 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6025 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6026 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6027 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6028 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6029 default: return Register();
6030 }
6031}
6032
6033// FastEmit functions for ISD::SDIV.
6034
6035Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6036 if (RetVT.SimpleTy != MVT::i32)
6037 return Register();
6038 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
6039 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
6040 }
6041 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
6042 return fastEmitInst_rr(MachineInstOpcode: ARM::SDIV, RC: &ARM::GPRRegClass, Op0, Op1);
6043 }
6044 return Register();
6045}
6046
6047Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6048 switch (VT.SimpleTy) {
6049 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
6050 default: return Register();
6051 }
6052}
6053
6054// FastEmit functions for ISD::SHL.
6055
6056Register fastEmit_ISD_SHL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6057 if (RetVT.SimpleTy != MVT::i32)
6058 return Register();
6059 if ((Subtarget->isThumb2())) {
6060 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSLrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6061 }
6062 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6063 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSLrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6064 }
6065 return Register();
6066}
6067
6068Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6069 switch (VT.SimpleTy) {
6070 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_rr(RetVT, Op0, Op1);
6071 default: return Register();
6072 }
6073}
6074
6075// FastEmit functions for ISD::SMAX.
6076
6077Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6078 if (RetVT.SimpleTy != MVT::v8i8)
6079 return Register();
6080 if ((Subtarget->hasNEON())) {
6081 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6082 }
6083 return Register();
6084}
6085
6086Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6087 if (RetVT.SimpleTy != MVT::v16i8)
6088 return Register();
6089 if ((Subtarget->hasMVEIntegerOps())) {
6090 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6091 }
6092 if ((Subtarget->hasNEON())) {
6093 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6094 }
6095 return Register();
6096}
6097
6098Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6099 if (RetVT.SimpleTy != MVT::v4i16)
6100 return Register();
6101 if ((Subtarget->hasNEON())) {
6102 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6103 }
6104 return Register();
6105}
6106
6107Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6108 if (RetVT.SimpleTy != MVT::v8i16)
6109 return Register();
6110 if ((Subtarget->hasMVEIntegerOps())) {
6111 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6112 }
6113 if ((Subtarget->hasNEON())) {
6114 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6115 }
6116 return Register();
6117}
6118
6119Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6120 if (RetVT.SimpleTy != MVT::v2i32)
6121 return Register();
6122 if ((Subtarget->hasNEON())) {
6123 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6124 }
6125 return Register();
6126}
6127
6128Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6129 if (RetVT.SimpleTy != MVT::v4i32)
6130 return Register();
6131 if ((Subtarget->hasMVEIntegerOps())) {
6132 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6133 }
6134 if ((Subtarget->hasNEON())) {
6135 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6136 }
6137 return Register();
6138}
6139
6140Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6141 switch (VT.SimpleTy) {
6142 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
6143 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
6144 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
6145 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
6146 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
6147 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
6148 default: return Register();
6149 }
6150}
6151
6152// FastEmit functions for ISD::SMIN.
6153
6154Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6155 if (RetVT.SimpleTy != MVT::v8i8)
6156 return Register();
6157 if ((Subtarget->hasNEON())) {
6158 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6159 }
6160 return Register();
6161}
6162
6163Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6164 if (RetVT.SimpleTy != MVT::v16i8)
6165 return Register();
6166 if ((Subtarget->hasMVEIntegerOps())) {
6167 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6168 }
6169 if ((Subtarget->hasNEON())) {
6170 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6171 }
6172 return Register();
6173}
6174
6175Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6176 if (RetVT.SimpleTy != MVT::v4i16)
6177 return Register();
6178 if ((Subtarget->hasNEON())) {
6179 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6180 }
6181 return Register();
6182}
6183
6184Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6185 if (RetVT.SimpleTy != MVT::v8i16)
6186 return Register();
6187 if ((Subtarget->hasMVEIntegerOps())) {
6188 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6189 }
6190 if ((Subtarget->hasNEON())) {
6191 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6192 }
6193 return Register();
6194}
6195
6196Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6197 if (RetVT.SimpleTy != MVT::v2i32)
6198 return Register();
6199 if ((Subtarget->hasNEON())) {
6200 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6201 }
6202 return Register();
6203}
6204
6205Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6206 if (RetVT.SimpleTy != MVT::v4i32)
6207 return Register();
6208 if ((Subtarget->hasMVEIntegerOps())) {
6209 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6210 }
6211 if ((Subtarget->hasNEON())) {
6212 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6213 }
6214 return Register();
6215}
6216
6217Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6218 switch (VT.SimpleTy) {
6219 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
6220 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
6221 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
6222 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
6223 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
6224 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
6225 default: return Register();
6226 }
6227}
6228
6229// FastEmit functions for ISD::SRA.
6230
6231Register fastEmit_ISD_SRA_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6232 if (RetVT.SimpleTy != MVT::i32)
6233 return Register();
6234 if ((Subtarget->isThumb2())) {
6235 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ASRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6236 }
6237 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6238 return fastEmitInst_rr(MachineInstOpcode: ARM::tASRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6239 }
6240 return Register();
6241}
6242
6243Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6244 switch (VT.SimpleTy) {
6245 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_rr(RetVT, Op0, Op1);
6246 default: return Register();
6247 }
6248}
6249
6250// FastEmit functions for ISD::SRL.
6251
6252Register fastEmit_ISD_SRL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6253 if (RetVT.SimpleTy != MVT::i32)
6254 return Register();
6255 if ((Subtarget->isThumb2())) {
6256 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6257 }
6258 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6259 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6260 }
6261 return Register();
6262}
6263
6264Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6265 switch (VT.SimpleTy) {
6266 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_rr(RetVT, Op0, Op1);
6267 default: return Register();
6268 }
6269}
6270
6271// FastEmit functions for ISD::SSUBSAT.
6272
6273Register fastEmit_ISD_SSUBSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6274 if (RetVT.SimpleTy != MVT::i32)
6275 return Register();
6276 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
6277 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB, RC: &ARM::rGPRRegClass, Op0, Op1);
6278 }
6279 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
6280 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6281 }
6282 return Register();
6283}
6284
6285Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6286 if (RetVT.SimpleTy != MVT::v8i8)
6287 return Register();
6288 if ((Subtarget->hasNEON())) {
6289 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6290 }
6291 return Register();
6292}
6293
6294Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6295 if (RetVT.SimpleTy != MVT::v16i8)
6296 return Register();
6297 if ((Subtarget->hasMVEIntegerOps())) {
6298 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6299 }
6300 if ((Subtarget->hasNEON())) {
6301 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6302 }
6303 return Register();
6304}
6305
6306Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6307 if (RetVT.SimpleTy != MVT::v4i16)
6308 return Register();
6309 if ((Subtarget->hasNEON())) {
6310 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6311 }
6312 return Register();
6313}
6314
6315Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6316 if (RetVT.SimpleTy != MVT::v8i16)
6317 return Register();
6318 if ((Subtarget->hasMVEIntegerOps())) {
6319 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6320 }
6321 if ((Subtarget->hasNEON())) {
6322 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6323 }
6324 return Register();
6325}
6326
6327Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6328 if (RetVT.SimpleTy != MVT::v2i32)
6329 return Register();
6330 if ((Subtarget->hasNEON())) {
6331 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6332 }
6333 return Register();
6334}
6335
6336Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6337 if (RetVT.SimpleTy != MVT::v4i32)
6338 return Register();
6339 if ((Subtarget->hasMVEIntegerOps())) {
6340 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6341 }
6342 if ((Subtarget->hasNEON())) {
6343 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6344 }
6345 return Register();
6346}
6347
6348Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6349 if (RetVT.SimpleTy != MVT::v1i64)
6350 return Register();
6351 if ((Subtarget->hasNEON())) {
6352 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6353 }
6354 return Register();
6355}
6356
6357Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6358 if (RetVT.SimpleTy != MVT::v2i64)
6359 return Register();
6360 if ((Subtarget->hasNEON())) {
6361 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6362 }
6363 return Register();
6364}
6365
6366Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6367 switch (VT.SimpleTy) {
6368 case MVT::i32: return fastEmit_ISD_SSUBSAT_MVT_i32_rr(RetVT, Op0, Op1);
6369 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6370 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6371 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6372 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6373 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6374 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6375 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6376 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6377 default: return Register();
6378 }
6379}
6380
6381// FastEmit functions for ISD::STRICT_FADD.
6382
6383Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6384 if (RetVT.SimpleTy != MVT::f16)
6385 return Register();
6386 if ((Subtarget->hasFullFP16())) {
6387 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
6388 }
6389 return Register();
6390}
6391
6392Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6393 if (RetVT.SimpleTy != MVT::f32)
6394 return Register();
6395 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6396 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
6397 }
6398 return Register();
6399}
6400
6401Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6402 if (RetVT.SimpleTy != MVT::f64)
6403 return Register();
6404 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6405 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
6406 }
6407 return Register();
6408}
6409
6410Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6411 switch (VT.SimpleTy) {
6412 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
6413 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
6414 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
6415 default: return Register();
6416 }
6417}
6418
6419// FastEmit functions for ISD::STRICT_FDIV.
6420
6421Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6422 if (RetVT.SimpleTy != MVT::f16)
6423 return Register();
6424 if ((Subtarget->hasFullFP16())) {
6425 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
6426 }
6427 return Register();
6428}
6429
6430Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6431 if (RetVT.SimpleTy != MVT::f32)
6432 return Register();
6433 if ((Subtarget->hasVFP2Base())) {
6434 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
6435 }
6436 return Register();
6437}
6438
6439Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6440 if (RetVT.SimpleTy != MVT::f64)
6441 return Register();
6442 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6443 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
6444 }
6445 return Register();
6446}
6447
6448Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6449 switch (VT.SimpleTy) {
6450 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
6451 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
6452 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
6453 default: return Register();
6454 }
6455}
6456
6457// FastEmit functions for ISD::STRICT_FMAXNUM.
6458
6459Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6460 if (RetVT.SimpleTy != MVT::f16)
6461 return Register();
6462 if ((Subtarget->hasFullFP16())) {
6463 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
6464 }
6465 return Register();
6466}
6467
6468Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6469 if (RetVT.SimpleTy != MVT::f32)
6470 return Register();
6471 if ((Subtarget->hasFPARMv8Base())) {
6472 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
6473 }
6474 return Register();
6475}
6476
6477Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6478 if (RetVT.SimpleTy != MVT::f64)
6479 return Register();
6480 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
6481 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
6482 }
6483 return Register();
6484}
6485
6486Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6487 switch (VT.SimpleTy) {
6488 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
6489 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
6490 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
6491 default: return Register();
6492 }
6493}
6494
6495// FastEmit functions for ISD::STRICT_FMINNUM.
6496
6497Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6498 if (RetVT.SimpleTy != MVT::f16)
6499 return Register();
6500 if ((Subtarget->hasFullFP16())) {
6501 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
6502 }
6503 return Register();
6504}
6505
6506Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6507 if (RetVT.SimpleTy != MVT::f32)
6508 return Register();
6509 if ((Subtarget->hasFPARMv8Base())) {
6510 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
6511 }
6512 return Register();
6513}
6514
6515Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6516 if (RetVT.SimpleTy != MVT::f64)
6517 return Register();
6518 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
6519 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
6520 }
6521 return Register();
6522}
6523
6524Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6525 switch (VT.SimpleTy) {
6526 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
6527 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
6528 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
6529 default: return Register();
6530 }
6531}
6532
6533// FastEmit functions for ISD::STRICT_FMUL.
6534
6535Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6536 if (RetVT.SimpleTy != MVT::f16)
6537 return Register();
6538 if ((Subtarget->hasFullFP16())) {
6539 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
6540 }
6541 return Register();
6542}
6543
6544Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6545 if (RetVT.SimpleTy != MVT::f32)
6546 return Register();
6547 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6548 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
6549 }
6550 return Register();
6551}
6552
6553Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6554 if (RetVT.SimpleTy != MVT::f64)
6555 return Register();
6556 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6557 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
6558 }
6559 return Register();
6560}
6561
6562Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6563 switch (VT.SimpleTy) {
6564 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
6565 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
6566 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
6567 default: return Register();
6568 }
6569}
6570
6571// FastEmit functions for ISD::STRICT_FSUB.
6572
6573Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6574 if (RetVT.SimpleTy != MVT::f16)
6575 return Register();
6576 if ((Subtarget->hasFullFP16())) {
6577 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
6578 }
6579 return Register();
6580}
6581
6582Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6583 if (RetVT.SimpleTy != MVT::f32)
6584 return Register();
6585 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6586 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
6587 }
6588 return Register();
6589}
6590
6591Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6592 if (RetVT.SimpleTy != MVT::f64)
6593 return Register();
6594 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6595 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
6596 }
6597 return Register();
6598}
6599
6600Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6601 switch (VT.SimpleTy) {
6602 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
6603 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
6604 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
6605 default: return Register();
6606 }
6607}
6608
6609// FastEmit functions for ISD::SUB.
6610
6611Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6612 if (RetVT.SimpleTy != MVT::i32)
6613 return Register();
6614 if ((Subtarget->isThumb2())) {
6615 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SUBrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6616 }
6617 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6618 return fastEmitInst_rr(MachineInstOpcode: ARM::tSUBrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6619 }
6620 if ((!Subtarget->isThumb())) {
6621 return fastEmitInst_rr(MachineInstOpcode: ARM::SUBrr, RC: &ARM::GPRRegClass, Op0, Op1);
6622 }
6623 return Register();
6624}
6625
6626Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6627 if (RetVT.SimpleTy != MVT::v8i8)
6628 return Register();
6629 if ((Subtarget->hasNEON())) {
6630 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6631 }
6632 return Register();
6633}
6634
6635Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6636 if (RetVT.SimpleTy != MVT::v16i8)
6637 return Register();
6638 if ((Subtarget->hasMVEIntegerOps())) {
6639 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi8, RC: &ARM::MQPRRegClass, Op0, Op1);
6640 }
6641 if ((Subtarget->hasNEON())) {
6642 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6643 }
6644 return Register();
6645}
6646
6647Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6648 if (RetVT.SimpleTy != MVT::v4i16)
6649 return Register();
6650 if ((Subtarget->hasNEON())) {
6651 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6652 }
6653 return Register();
6654}
6655
6656Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6657 if (RetVT.SimpleTy != MVT::v8i16)
6658 return Register();
6659 if ((Subtarget->hasMVEIntegerOps())) {
6660 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi16, RC: &ARM::MQPRRegClass, Op0, Op1);
6661 }
6662 if ((Subtarget->hasNEON())) {
6663 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6664 }
6665 return Register();
6666}
6667
6668Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6669 if (RetVT.SimpleTy != MVT::v2i32)
6670 return Register();
6671 if ((Subtarget->hasNEON())) {
6672 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6673 }
6674 return Register();
6675}
6676
6677Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6678 if (RetVT.SimpleTy != MVT::v4i32)
6679 return Register();
6680 if ((Subtarget->hasMVEIntegerOps())) {
6681 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi32, RC: &ARM::MQPRRegClass, Op0, Op1);
6682 }
6683 if ((Subtarget->hasNEON())) {
6684 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6685 }
6686 return Register();
6687}
6688
6689Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6690 if (RetVT.SimpleTy != MVT::v1i64)
6691 return Register();
6692 if ((Subtarget->hasNEON())) {
6693 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6694 }
6695 return Register();
6696}
6697
6698Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6699 if (RetVT.SimpleTy != MVT::v2i64)
6700 return Register();
6701 if ((Subtarget->hasNEON())) {
6702 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6703 }
6704 return Register();
6705}
6706
6707Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6708 switch (VT.SimpleTy) {
6709 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
6710 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
6711 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
6712 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
6713 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
6714 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
6715 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
6716 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
6717 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
6718 default: return Register();
6719 }
6720}
6721
6722// FastEmit functions for ISD::UADDSAT.
6723
6724Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6725 if (RetVT.SimpleTy != MVT::v8i8)
6726 return Register();
6727 if ((Subtarget->hasNEON())) {
6728 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6729 }
6730 return Register();
6731}
6732
6733Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6734 if (RetVT.SimpleTy != MVT::v16i8)
6735 return Register();
6736 if ((Subtarget->hasMVEIntegerOps())) {
6737 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6738 }
6739 if ((Subtarget->hasNEON())) {
6740 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6741 }
6742 return Register();
6743}
6744
6745Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6746 if (RetVT.SimpleTy != MVT::v4i16)
6747 return Register();
6748 if ((Subtarget->hasNEON())) {
6749 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6750 }
6751 return Register();
6752}
6753
6754Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6755 if (RetVT.SimpleTy != MVT::v8i16)
6756 return Register();
6757 if ((Subtarget->hasMVEIntegerOps())) {
6758 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6759 }
6760 if ((Subtarget->hasNEON())) {
6761 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6762 }
6763 return Register();
6764}
6765
6766Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6767 if (RetVT.SimpleTy != MVT::v2i32)
6768 return Register();
6769 if ((Subtarget->hasNEON())) {
6770 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6771 }
6772 return Register();
6773}
6774
6775Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6776 if (RetVT.SimpleTy != MVT::v4i32)
6777 return Register();
6778 if ((Subtarget->hasMVEIntegerOps())) {
6779 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6780 }
6781 if ((Subtarget->hasNEON())) {
6782 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6783 }
6784 return Register();
6785}
6786
6787Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6788 if (RetVT.SimpleTy != MVT::v1i64)
6789 return Register();
6790 if ((Subtarget->hasNEON())) {
6791 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6792 }
6793 return Register();
6794}
6795
6796Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6797 if (RetVT.SimpleTy != MVT::v2i64)
6798 return Register();
6799 if ((Subtarget->hasNEON())) {
6800 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6801 }
6802 return Register();
6803}
6804
6805Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6806 switch (VT.SimpleTy) {
6807 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6808 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6809 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6810 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6811 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6812 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6813 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6814 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6815 default: return Register();
6816 }
6817}
6818
6819// FastEmit functions for ISD::UDIV.
6820
6821Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6822 if (RetVT.SimpleTy != MVT::i32)
6823 return Register();
6824 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
6825 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
6826 }
6827 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
6828 return fastEmitInst_rr(MachineInstOpcode: ARM::UDIV, RC: &ARM::GPRRegClass, Op0, Op1);
6829 }
6830 return Register();
6831}
6832
6833Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6834 switch (VT.SimpleTy) {
6835 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
6836 default: return Register();
6837 }
6838}
6839
6840// FastEmit functions for ISD::UMAX.
6841
6842Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6843 if (RetVT.SimpleTy != MVT::v8i8)
6844 return Register();
6845 if ((Subtarget->hasNEON())) {
6846 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6847 }
6848 return Register();
6849}
6850
6851Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6852 if (RetVT.SimpleTy != MVT::v16i8)
6853 return Register();
6854 if ((Subtarget->hasMVEIntegerOps())) {
6855 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6856 }
6857 if ((Subtarget->hasNEON())) {
6858 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6859 }
6860 return Register();
6861}
6862
6863Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6864 if (RetVT.SimpleTy != MVT::v4i16)
6865 return Register();
6866 if ((Subtarget->hasNEON())) {
6867 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6868 }
6869 return Register();
6870}
6871
6872Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6873 if (RetVT.SimpleTy != MVT::v8i16)
6874 return Register();
6875 if ((Subtarget->hasMVEIntegerOps())) {
6876 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6877 }
6878 if ((Subtarget->hasNEON())) {
6879 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6880 }
6881 return Register();
6882}
6883
6884Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6885 if (RetVT.SimpleTy != MVT::v2i32)
6886 return Register();
6887 if ((Subtarget->hasNEON())) {
6888 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6889 }
6890 return Register();
6891}
6892
6893Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6894 if (RetVT.SimpleTy != MVT::v4i32)
6895 return Register();
6896 if ((Subtarget->hasMVEIntegerOps())) {
6897 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6898 }
6899 if ((Subtarget->hasNEON())) {
6900 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6901 }
6902 return Register();
6903}
6904
6905Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6906 switch (VT.SimpleTy) {
6907 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
6908 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
6909 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
6910 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
6911 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
6912 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
6913 default: return Register();
6914 }
6915}
6916
6917// FastEmit functions for ISD::UMIN.
6918
6919Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6920 if (RetVT.SimpleTy != MVT::v8i8)
6921 return Register();
6922 if ((Subtarget->hasNEON())) {
6923 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6924 }
6925 return Register();
6926}
6927
6928Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6929 if (RetVT.SimpleTy != MVT::v16i8)
6930 return Register();
6931 if ((Subtarget->hasMVEIntegerOps())) {
6932 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6933 }
6934 if ((Subtarget->hasNEON())) {
6935 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6936 }
6937 return Register();
6938}
6939
6940Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6941 if (RetVT.SimpleTy != MVT::v4i16)
6942 return Register();
6943 if ((Subtarget->hasNEON())) {
6944 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6945 }
6946 return Register();
6947}
6948
6949Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6950 if (RetVT.SimpleTy != MVT::v8i16)
6951 return Register();
6952 if ((Subtarget->hasMVEIntegerOps())) {
6953 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6954 }
6955 if ((Subtarget->hasNEON())) {
6956 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6957 }
6958 return Register();
6959}
6960
6961Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6962 if (RetVT.SimpleTy != MVT::v2i32)
6963 return Register();
6964 if ((Subtarget->hasNEON())) {
6965 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6966 }
6967 return Register();
6968}
6969
6970Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6971 if (RetVT.SimpleTy != MVT::v4i32)
6972 return Register();
6973 if ((Subtarget->hasMVEIntegerOps())) {
6974 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6975 }
6976 if ((Subtarget->hasNEON())) {
6977 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6978 }
6979 return Register();
6980}
6981
6982Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6983 switch (VT.SimpleTy) {
6984 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
6985 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
6986 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
6987 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
6988 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
6989 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
6990 default: return Register();
6991 }
6992}
6993
6994// FastEmit functions for ISD::USUBSAT.
6995
6996Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6997 if (RetVT.SimpleTy != MVT::v8i8)
6998 return Register();
6999 if ((Subtarget->hasNEON())) {
7000 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7001 }
7002 return Register();
7003}
7004
7005Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7006 if (RetVT.SimpleTy != MVT::v16i8)
7007 return Register();
7008 if ((Subtarget->hasMVEIntegerOps())) {
7009 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu8, RC: &ARM::MQPRRegClass, Op0, Op1);
7010 }
7011 if ((Subtarget->hasNEON())) {
7012 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7013 }
7014 return Register();
7015}
7016
7017Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7018 if (RetVT.SimpleTy != MVT::v4i16)
7019 return Register();
7020 if ((Subtarget->hasNEON())) {
7021 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7022 }
7023 return Register();
7024}
7025
7026Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7027 if (RetVT.SimpleTy != MVT::v8i16)
7028 return Register();
7029 if ((Subtarget->hasMVEIntegerOps())) {
7030 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7031 }
7032 if ((Subtarget->hasNEON())) {
7033 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7034 }
7035 return Register();
7036}
7037
7038Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7039 if (RetVT.SimpleTy != MVT::v2i32)
7040 return Register();
7041 if ((Subtarget->hasNEON())) {
7042 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7043 }
7044 return Register();
7045}
7046
7047Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7048 if (RetVT.SimpleTy != MVT::v4i32)
7049 return Register();
7050 if ((Subtarget->hasMVEIntegerOps())) {
7051 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7052 }
7053 if ((Subtarget->hasNEON())) {
7054 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7055 }
7056 return Register();
7057}
7058
7059Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7060 if (RetVT.SimpleTy != MVT::v1i64)
7061 return Register();
7062 if ((Subtarget->hasNEON())) {
7063 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
7064 }
7065 return Register();
7066}
7067
7068Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7069 if (RetVT.SimpleTy != MVT::v2i64)
7070 return Register();
7071 if ((Subtarget->hasNEON())) {
7072 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
7073 }
7074 return Register();
7075}
7076
7077Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7078 switch (VT.SimpleTy) {
7079 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
7080 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
7081 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
7082 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
7083 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
7084 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
7085 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
7086 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
7087 default: return Register();
7088 }
7089}
7090
7091// FastEmit functions for ISD::XOR.
7092
7093Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
7094 if (RetVT.SimpleTy != MVT::i32)
7095 return Register();
7096 if ((Subtarget->isThumb2())) {
7097 return fastEmitInst_rr(MachineInstOpcode: ARM::t2EORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
7098 }
7099 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7100 return fastEmitInst_rr(MachineInstOpcode: ARM::tEOR, RC: &ARM::tGPRRegClass, Op0, Op1);
7101 }
7102 if ((!Subtarget->isThumb())) {
7103 return fastEmitInst_rr(MachineInstOpcode: ARM::EORrr, RC: &ARM::GPRRegClass, Op0, Op1);
7104 }
7105 return Register();
7106}
7107
7108Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7109 if (RetVT.SimpleTy != MVT::v8i8)
7110 return Register();
7111 if ((Subtarget->hasNEON())) {
7112 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7113 }
7114 return Register();
7115}
7116
7117Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7118 if (RetVT.SimpleTy != MVT::v16i8)
7119 return Register();
7120 if ((Subtarget->hasMVEIntegerOps())) {
7121 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7122 }
7123 if ((Subtarget->hasNEON())) {
7124 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7125 }
7126 return Register();
7127}
7128
7129Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7130 if (RetVT.SimpleTy != MVT::v4i16)
7131 return Register();
7132 if ((Subtarget->hasNEON())) {
7133 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7134 }
7135 return Register();
7136}
7137
7138Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7139 if (RetVT.SimpleTy != MVT::v8i16)
7140 return Register();
7141 if ((Subtarget->hasMVEIntegerOps())) {
7142 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7143 }
7144 if ((Subtarget->hasNEON())) {
7145 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7146 }
7147 return Register();
7148}
7149
7150Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7151 if (RetVT.SimpleTy != MVT::v2i32)
7152 return Register();
7153 if ((Subtarget->hasNEON())) {
7154 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7155 }
7156 return Register();
7157}
7158
7159Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7160 if (RetVT.SimpleTy != MVT::v4i32)
7161 return Register();
7162 if ((Subtarget->hasMVEIntegerOps())) {
7163 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7164 }
7165 if ((Subtarget->hasNEON())) {
7166 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7167 }
7168 return Register();
7169}
7170
7171Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7172 if (RetVT.SimpleTy != MVT::v1i64)
7173 return Register();
7174 if ((Subtarget->hasNEON())) {
7175 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7176 }
7177 return Register();
7178}
7179
7180Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7181 if (RetVT.SimpleTy != MVT::v2i64)
7182 return Register();
7183 if ((Subtarget->hasMVEIntegerOps())) {
7184 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7185 }
7186 if ((Subtarget->hasNEON())) {
7187 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7188 }
7189 return Register();
7190}
7191
7192Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7193 switch (VT.SimpleTy) {
7194 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
7195 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
7196 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
7197 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
7198 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
7199 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
7200 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
7201 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
7202 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
7203 default: return Register();
7204 }
7205}
7206
7207// Top-level FastEmit function.
7208
7209Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
7210 switch (Opcode) {
7211 case ARMISD::QADD16b: return fastEmit_ARMISD_QADD16b_rr(VT, RetVT, Op0, Op1);
7212 case ARMISD::QADD8b: return fastEmit_ARMISD_QADD8b_rr(VT, RetVT, Op0, Op1);
7213 case ARMISD::QSUB16b: return fastEmit_ARMISD_QSUB16b_rr(VT, RetVT, Op0, Op1);
7214 case ARMISD::QSUB8b: return fastEmit_ARMISD_QSUB8b_rr(VT, RetVT, Op0, Op1);
7215 case ARMISD::SMULWB: return fastEmit_ARMISD_SMULWB_rr(VT, RetVT, Op0, Op1);
7216 case ARMISD::SMULWT: return fastEmit_ARMISD_SMULWT_rr(VT, RetVT, Op0, Op1);
7217 case ARMISD::UQADD16b: return fastEmit_ARMISD_UQADD16b_rr(VT, RetVT, Op0, Op1);
7218 case ARMISD::UQADD8b: return fastEmit_ARMISD_UQADD8b_rr(VT, RetVT, Op0, Op1);
7219 case ARMISD::UQSUB16b: return fastEmit_ARMISD_UQSUB16b_rr(VT, RetVT, Op0, Op1);
7220 case ARMISD::UQSUB8b: return fastEmit_ARMISD_UQSUB8b_rr(VT, RetVT, Op0, Op1);
7221 case ARMISD::VMLAVs: return fastEmit_ARMISD_VMLAVs_rr(VT, RetVT, Op0, Op1);
7222 case ARMISD::VMLAVu: return fastEmit_ARMISD_VMLAVu_rr(VT, RetVT, Op0, Op1);
7223 case ARMISD::VMOVDRR: return fastEmit_ARMISD_VMOVDRR_rr(VT, RetVT, Op0, Op1);
7224 case ARMISD::VMULLs: return fastEmit_ARMISD_VMULLs_rr(VT, RetVT, Op0, Op1);
7225 case ARMISD::VMULLu: return fastEmit_ARMISD_VMULLu_rr(VT, RetVT, Op0, Op1);
7226 case ARMISD::VQDMULH: return fastEmit_ARMISD_VQDMULH_rr(VT, RetVT, Op0, Op1);
7227 case ARMISD::VSHLs: return fastEmit_ARMISD_VSHLs_rr(VT, RetVT, Op0, Op1);
7228 case ARMISD::VSHLu: return fastEmit_ARMISD_VSHLu_rr(VT, RetVT, Op0, Op1);
7229 case ARMISD::VTBL1: return fastEmit_ARMISD_VTBL1_rr(VT, RetVT, Op0, Op1);
7230 case ARMISD::VTST: return fastEmit_ARMISD_VTST_rr(VT, RetVT, Op0, Op1);
7231 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
7232 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
7233 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
7234 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
7235 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
7236 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
7237 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
7238 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
7239 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
7240 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
7241 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
7242 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
7243 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
7244 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
7245 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
7246 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
7247 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
7248 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
7249 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
7250 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
7251 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
7252 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
7253 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
7254 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
7255 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
7256 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
7257 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
7258 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
7259 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
7260 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
7261 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
7262 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
7263 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
7264 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
7265 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
7266 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
7267 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
7268 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
7269 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
7270 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
7271 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
7272 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
7273 default: return Register();
7274 }
7275}
7276
7277// FastEmit functions for ARMISD::PIC_ADD.
7278
7279Register fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7280 if (RetVT.SimpleTy != MVT::i32)
7281 return Register();
7282 if ((Subtarget->isThumb())) {
7283 return fastEmitInst_ri(MachineInstOpcode: ARM::tPICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7284 }
7285 if ((!Subtarget->isThumb())) {
7286 return fastEmitInst_ri(MachineInstOpcode: ARM::PICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7287 }
7288 return Register();
7289}
7290
7291Register fastEmit_ARMISD_PIC_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7292 switch (VT.SimpleTy) {
7293 case MVT::i32: return fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(RetVT, Op0, imm1);
7294 default: return Register();
7295 }
7296}
7297
7298// FastEmit functions for ARMISD::VDUPLANE.
7299
7300Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7301 if (RetVT.SimpleTy != MVT::v8i8)
7302 return Register();
7303 if ((Subtarget->hasNEON())) {
7304 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7305 }
7306 return Register();
7307}
7308
7309Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7310 if (RetVT.SimpleTy != MVT::v4i16)
7311 return Register();
7312 if ((Subtarget->hasNEON())) {
7313 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7314 }
7315 return Register();
7316}
7317
7318Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7319 if (RetVT.SimpleTy != MVT::v2i32)
7320 return Register();
7321 if ((Subtarget->hasNEON())) {
7322 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7323 }
7324 return Register();
7325}
7326
7327Register fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7328 if (RetVT.SimpleTy != MVT::v4f16)
7329 return Register();
7330 if ((Subtarget->hasNEON())) {
7331 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7332 }
7333 return Register();
7334}
7335
7336Register fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7337 if (RetVT.SimpleTy != MVT::v4bf16)
7338 return Register();
7339 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
7340 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7341 }
7342 return Register();
7343}
7344
7345Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Register Op0, uint64_t imm1) {
7346 if ((Subtarget->hasNEON())) {
7347 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7348 }
7349 return Register();
7350}
7351
7352Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Register Op0, uint64_t imm1) {
7353 if ((Subtarget->hasNEON())) {
7354 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7355 }
7356 return Register();
7357}
7358
7359Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7360switch (RetVT.SimpleTy) {
7361 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Op0, imm1);
7362 case MVT::v4f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Op0, imm1);
7363 default: return Register();
7364}
7365}
7366
7367Register fastEmit_ARMISD_VDUPLANE_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7368 switch (VT.SimpleTy) {
7369 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(RetVT, Op0, imm1);
7370 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(RetVT, Op0, imm1);
7371 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(RetVT, Op0, imm1);
7372 case MVT::v4f16: return fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(RetVT, Op0, imm1);
7373 case MVT::v4bf16: return fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(RetVT, Op0, imm1);
7374 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(RetVT, Op0, imm1);
7375 default: return Register();
7376 }
7377}
7378
7379// FastEmit functions for ARMISD::VGETLANEs.
7380
7381Register fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7382 if (RetVT.SimpleTy != MVT::i32)
7383 return Register();
7384 if ((Subtarget->hasNEON())) {
7385 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7386 }
7387 return Register();
7388}
7389
7390Register fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7391 if (RetVT.SimpleTy != MVT::i32)
7392 return Register();
7393 if ((Subtarget->hasMVEIntegerOps())) {
7394 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7395 }
7396 return Register();
7397}
7398
7399Register fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7400 if (RetVT.SimpleTy != MVT::i32)
7401 return Register();
7402 if ((Subtarget->hasNEON())) {
7403 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7404 }
7405 return Register();
7406}
7407
7408Register fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7409 if (RetVT.SimpleTy != MVT::i32)
7410 return Register();
7411 if ((Subtarget->hasMVEIntegerOps())) {
7412 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7413 }
7414 return Register();
7415}
7416
7417Register fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7418 if (RetVT.SimpleTy != MVT::i32)
7419 return Register();
7420 if ((Subtarget->hasMVEIntegerOps())) {
7421 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7422 }
7423 return Register();
7424}
7425
7426Register fastEmit_ARMISD_VGETLANEs_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7427 switch (VT.SimpleTy) {
7428 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(RetVT, Op0, imm1);
7429 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(RetVT, Op0, imm1);
7430 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(RetVT, Op0, imm1);
7431 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(RetVT, Op0, imm1);
7432 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(RetVT, Op0, imm1);
7433 default: return Register();
7434 }
7435}
7436
7437// FastEmit functions for ARMISD::VGETLANEu.
7438
7439Register fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7440 if (RetVT.SimpleTy != MVT::i32)
7441 return Register();
7442 if ((Subtarget->hasNEON())) {
7443 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7444 }
7445 return Register();
7446}
7447
7448Register fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7449 if (RetVT.SimpleTy != MVT::i32)
7450 return Register();
7451 if ((Subtarget->hasMVEIntegerOps())) {
7452 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7453 }
7454 return Register();
7455}
7456
7457Register fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7458 if (RetVT.SimpleTy != MVT::i32)
7459 return Register();
7460 if ((Subtarget->hasNEON())) {
7461 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7462 }
7463 return Register();
7464}
7465
7466Register fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7467 if (RetVT.SimpleTy != MVT::i32)
7468 return Register();
7469 if ((Subtarget->hasMVEIntegerOps())) {
7470 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7471 }
7472 return Register();
7473}
7474
7475Register fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7476 if (RetVT.SimpleTy != MVT::i32)
7477 return Register();
7478 if ((Subtarget->hasNEON())) {
7479 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7480 }
7481 return Register();
7482}
7483
7484Register fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7485 if (RetVT.SimpleTy != MVT::i32)
7486 return Register();
7487 if ((Subtarget->hasMVEIntegerOps())) {
7488 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7489 }
7490 return Register();
7491}
7492
7493Register fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7494 if (RetVT.SimpleTy != MVT::i32)
7495 return Register();
7496 if ((Subtarget->hasNEON())) {
7497 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7498 }
7499 return Register();
7500}
7501
7502Register fastEmit_ARMISD_VGETLANEu_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7503 switch (VT.SimpleTy) {
7504 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(RetVT, Op0, imm1);
7505 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(RetVT, Op0, imm1);
7506 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(RetVT, Op0, imm1);
7507 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(RetVT, Op0, imm1);
7508 case MVT::v4f16: return fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(RetVT, Op0, imm1);
7509 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(RetVT, Op0, imm1);
7510 case MVT::v4bf16: return fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(RetVT, Op0, imm1);
7511 default: return Register();
7512 }
7513}
7514
7515// FastEmit functions for ARMISD::VQSHLsIMM.
7516
7517Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7518 if (RetVT.SimpleTy != MVT::v8i8)
7519 return Register();
7520 if ((Subtarget->hasNEON())) {
7521 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7522 }
7523 return Register();
7524}
7525
7526Register fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7527 if (RetVT.SimpleTy != MVT::v16i8)
7528 return Register();
7529 if ((Subtarget->hasNEON())) {
7530 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7531 }
7532 return Register();
7533}
7534
7535Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7536 if (RetVT.SimpleTy != MVT::v4i16)
7537 return Register();
7538 if ((Subtarget->hasNEON())) {
7539 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7540 }
7541 return Register();
7542}
7543
7544Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7545 if (RetVT.SimpleTy != MVT::v8i16)
7546 return Register();
7547 if ((Subtarget->hasNEON())) {
7548 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7549 }
7550 return Register();
7551}
7552
7553Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7554 if (RetVT.SimpleTy != MVT::v2i32)
7555 return Register();
7556 if ((Subtarget->hasNEON())) {
7557 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7558 }
7559 return Register();
7560}
7561
7562Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7563 if (RetVT.SimpleTy != MVT::v4i32)
7564 return Register();
7565 if ((Subtarget->hasNEON())) {
7566 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7567 }
7568 return Register();
7569}
7570
7571Register fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7572 if (RetVT.SimpleTy != MVT::v1i64)
7573 return Register();
7574 if ((Subtarget->hasNEON())) {
7575 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7576 }
7577 return Register();
7578}
7579
7580Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7581 if (RetVT.SimpleTy != MVT::v2i64)
7582 return Register();
7583 if ((Subtarget->hasNEON())) {
7584 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7585 }
7586 return Register();
7587}
7588
7589Register fastEmit_ARMISD_VQSHLsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7590 switch (VT.SimpleTy) {
7591 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7592 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7593 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7594 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7595 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7596 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7597 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7598 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7599 default: return Register();
7600 }
7601}
7602
7603// FastEmit functions for ARMISD::VQSHLsuIMM.
7604
7605Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7606 if (RetVT.SimpleTy != MVT::v8i8)
7607 return Register();
7608 if ((Subtarget->hasNEON())) {
7609 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7610 }
7611 return Register();
7612}
7613
7614Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7615 if (RetVT.SimpleTy != MVT::v16i8)
7616 return Register();
7617 if ((Subtarget->hasNEON())) {
7618 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7619 }
7620 return Register();
7621}
7622
7623Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7624 if (RetVT.SimpleTy != MVT::v4i16)
7625 return Register();
7626 if ((Subtarget->hasNEON())) {
7627 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7628 }
7629 return Register();
7630}
7631
7632Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7633 if (RetVT.SimpleTy != MVT::v8i16)
7634 return Register();
7635 if ((Subtarget->hasNEON())) {
7636 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7637 }
7638 return Register();
7639}
7640
7641Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7642 if (RetVT.SimpleTy != MVT::v2i32)
7643 return Register();
7644 if ((Subtarget->hasNEON())) {
7645 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7646 }
7647 return Register();
7648}
7649
7650Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7651 if (RetVT.SimpleTy != MVT::v4i32)
7652 return Register();
7653 if ((Subtarget->hasNEON())) {
7654 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7655 }
7656 return Register();
7657}
7658
7659Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7660 if (RetVT.SimpleTy != MVT::v1i64)
7661 return Register();
7662 if ((Subtarget->hasNEON())) {
7663 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7664 }
7665 return Register();
7666}
7667
7668Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7669 if (RetVT.SimpleTy != MVT::v2i64)
7670 return Register();
7671 if ((Subtarget->hasNEON())) {
7672 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7673 }
7674 return Register();
7675}
7676
7677Register fastEmit_ARMISD_VQSHLsuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7678 switch (VT.SimpleTy) {
7679 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7680 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7681 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7682 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7683 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7684 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7685 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7686 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7687 default: return Register();
7688 }
7689}
7690
7691// FastEmit functions for ARMISD::VQSHLuIMM.
7692
7693Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7694 if (RetVT.SimpleTy != MVT::v8i8)
7695 return Register();
7696 if ((Subtarget->hasNEON())) {
7697 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7698 }
7699 return Register();
7700}
7701
7702Register fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7703 if (RetVT.SimpleTy != MVT::v16i8)
7704 return Register();
7705 if ((Subtarget->hasNEON())) {
7706 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7707 }
7708 return Register();
7709}
7710
7711Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7712 if (RetVT.SimpleTy != MVT::v4i16)
7713 return Register();
7714 if ((Subtarget->hasNEON())) {
7715 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7716 }
7717 return Register();
7718}
7719
7720Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7721 if (RetVT.SimpleTy != MVT::v8i16)
7722 return Register();
7723 if ((Subtarget->hasNEON())) {
7724 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7725 }
7726 return Register();
7727}
7728
7729Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7730 if (RetVT.SimpleTy != MVT::v2i32)
7731 return Register();
7732 if ((Subtarget->hasNEON())) {
7733 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7734 }
7735 return Register();
7736}
7737
7738Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7739 if (RetVT.SimpleTy != MVT::v4i32)
7740 return Register();
7741 if ((Subtarget->hasNEON())) {
7742 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7743 }
7744 return Register();
7745}
7746
7747Register fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7748 if (RetVT.SimpleTy != MVT::v1i64)
7749 return Register();
7750 if ((Subtarget->hasNEON())) {
7751 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7752 }
7753 return Register();
7754}
7755
7756Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7757 if (RetVT.SimpleTy != MVT::v2i64)
7758 return Register();
7759 if ((Subtarget->hasNEON())) {
7760 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7761 }
7762 return Register();
7763}
7764
7765Register fastEmit_ARMISD_VQSHLuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7766 switch (VT.SimpleTy) {
7767 case MVT::v8i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7768 case MVT::v16i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7769 case MVT::v4i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7770 case MVT::v8i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7771 case MVT::v2i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7772 case MVT::v4i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7773 case MVT::v1i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7774 case MVT::v2i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7775 default: return Register();
7776 }
7777}
7778
7779// FastEmit functions for ARMISD::VRSHRsIMM.
7780
7781Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7782 if (RetVT.SimpleTy != MVT::v8i8)
7783 return Register();
7784 if ((Subtarget->hasNEON())) {
7785 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7786 }
7787 return Register();
7788}
7789
7790Register fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7791 if (RetVT.SimpleTy != MVT::v16i8)
7792 return Register();
7793 if ((Subtarget->hasNEON())) {
7794 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7795 }
7796 return Register();
7797}
7798
7799Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7800 if (RetVT.SimpleTy != MVT::v4i16)
7801 return Register();
7802 if ((Subtarget->hasNEON())) {
7803 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7804 }
7805 return Register();
7806}
7807
7808Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7809 if (RetVT.SimpleTy != MVT::v8i16)
7810 return Register();
7811 if ((Subtarget->hasNEON())) {
7812 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7813 }
7814 return Register();
7815}
7816
7817Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7818 if (RetVT.SimpleTy != MVT::v2i32)
7819 return Register();
7820 if ((Subtarget->hasNEON())) {
7821 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7822 }
7823 return Register();
7824}
7825
7826Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7827 if (RetVT.SimpleTy != MVT::v4i32)
7828 return Register();
7829 if ((Subtarget->hasNEON())) {
7830 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7831 }
7832 return Register();
7833}
7834
7835Register fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7836 if (RetVT.SimpleTy != MVT::v1i64)
7837 return Register();
7838 if ((Subtarget->hasNEON())) {
7839 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7840 }
7841 return Register();
7842}
7843
7844Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7845 if (RetVT.SimpleTy != MVT::v2i64)
7846 return Register();
7847 if ((Subtarget->hasNEON())) {
7848 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7849 }
7850 return Register();
7851}
7852
7853Register fastEmit_ARMISD_VRSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7854 switch (VT.SimpleTy) {
7855 case MVT::v8i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7856 case MVT::v16i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7857 case MVT::v4i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7858 case MVT::v8i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7859 case MVT::v2i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7860 case MVT::v4i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7861 case MVT::v1i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7862 case MVT::v2i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7863 default: return Register();
7864 }
7865}
7866
7867// FastEmit functions for ARMISD::VRSHRuIMM.
7868
7869Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7870 if (RetVT.SimpleTy != MVT::v8i8)
7871 return Register();
7872 if ((Subtarget->hasNEON())) {
7873 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7874 }
7875 return Register();
7876}
7877
7878Register fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7879 if (RetVT.SimpleTy != MVT::v16i8)
7880 return Register();
7881 if ((Subtarget->hasNEON())) {
7882 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7883 }
7884 return Register();
7885}
7886
7887Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7888 if (RetVT.SimpleTy != MVT::v4i16)
7889 return Register();
7890 if ((Subtarget->hasNEON())) {
7891 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7892 }
7893 return Register();
7894}
7895
7896Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7897 if (RetVT.SimpleTy != MVT::v8i16)
7898 return Register();
7899 if ((Subtarget->hasNEON())) {
7900 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7901 }
7902 return Register();
7903}
7904
7905Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7906 if (RetVT.SimpleTy != MVT::v2i32)
7907 return Register();
7908 if ((Subtarget->hasNEON())) {
7909 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7910 }
7911 return Register();
7912}
7913
7914Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7915 if (RetVT.SimpleTy != MVT::v4i32)
7916 return Register();
7917 if ((Subtarget->hasNEON())) {
7918 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7919 }
7920 return Register();
7921}
7922
7923Register fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7924 if (RetVT.SimpleTy != MVT::v1i64)
7925 return Register();
7926 if ((Subtarget->hasNEON())) {
7927 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7928 }
7929 return Register();
7930}
7931
7932Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7933 if (RetVT.SimpleTy != MVT::v2i64)
7934 return Register();
7935 if ((Subtarget->hasNEON())) {
7936 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7937 }
7938 return Register();
7939}
7940
7941Register fastEmit_ARMISD_VRSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7942 switch (VT.SimpleTy) {
7943 case MVT::v8i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7944 case MVT::v16i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7945 case MVT::v4i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7946 case MVT::v8i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7947 case MVT::v2i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7948 case MVT::v4i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7949 case MVT::v1i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7950 case MVT::v2i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7951 default: return Register();
7952 }
7953}
7954
7955// FastEmit functions for ARMISD::VSHLIMM.
7956
7957Register fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7958 if (RetVT.SimpleTy != MVT::v8i8)
7959 return Register();
7960 if ((Subtarget->hasNEON())) {
7961 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7962 }
7963 return Register();
7964}
7965
7966Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7967 if (RetVT.SimpleTy != MVT::v16i8)
7968 return Register();
7969 if ((Subtarget->hasNEON())) {
7970 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7971 }
7972 return Register();
7973}
7974
7975Register fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7976 if (RetVT.SimpleTy != MVT::v4i16)
7977 return Register();
7978 if ((Subtarget->hasNEON())) {
7979 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7980 }
7981 return Register();
7982}
7983
7984Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7985 if (RetVT.SimpleTy != MVT::v8i16)
7986 return Register();
7987 if ((Subtarget->hasNEON())) {
7988 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7989 }
7990 return Register();
7991}
7992
7993Register fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7994 if (RetVT.SimpleTy != MVT::v2i32)
7995 return Register();
7996 if ((Subtarget->hasNEON())) {
7997 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7998 }
7999 return Register();
8000}
8001
8002Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8003 if (RetVT.SimpleTy != MVT::v4i32)
8004 return Register();
8005 if ((Subtarget->hasNEON())) {
8006 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8007 }
8008 return Register();
8009}
8010
8011Register fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8012 if (RetVT.SimpleTy != MVT::v1i64)
8013 return Register();
8014 if ((Subtarget->hasNEON())) {
8015 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8016 }
8017 return Register();
8018}
8019
8020Register fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8021 if (RetVT.SimpleTy != MVT::v2i64)
8022 return Register();
8023 if ((Subtarget->hasNEON())) {
8024 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8025 }
8026 return Register();
8027}
8028
8029Register fastEmit_ARMISD_VSHLIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8030 switch (VT.SimpleTy) {
8031 case MVT::v8i8: return fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8032 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8033 case MVT::v4i16: return fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8034 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8035 case MVT::v2i32: return fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8036 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8037 case MVT::v1i64: return fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8038 case MVT::v2i64: return fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8039 default: return Register();
8040 }
8041}
8042
8043// FastEmit functions for ARMISD::VSHRsIMM.
8044
8045Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8046 if (RetVT.SimpleTy != MVT::v8i8)
8047 return Register();
8048 if ((Subtarget->hasNEON())) {
8049 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8050 }
8051 return Register();
8052}
8053
8054Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8055 if (RetVT.SimpleTy != MVT::v16i8)
8056 return Register();
8057 if ((Subtarget->hasNEON())) {
8058 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8059 }
8060 return Register();
8061}
8062
8063Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8064 if (RetVT.SimpleTy != MVT::v4i16)
8065 return Register();
8066 if ((Subtarget->hasNEON())) {
8067 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8068 }
8069 return Register();
8070}
8071
8072Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8073 if (RetVT.SimpleTy != MVT::v8i16)
8074 return Register();
8075 if ((Subtarget->hasNEON())) {
8076 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8077 }
8078 return Register();
8079}
8080
8081Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8082 if (RetVT.SimpleTy != MVT::v2i32)
8083 return Register();
8084 if ((Subtarget->hasNEON())) {
8085 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8086 }
8087 return Register();
8088}
8089
8090Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8091 if (RetVT.SimpleTy != MVT::v4i32)
8092 return Register();
8093 if ((Subtarget->hasNEON())) {
8094 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8095 }
8096 return Register();
8097}
8098
8099Register fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8100 if (RetVT.SimpleTy != MVT::v1i64)
8101 return Register();
8102 if ((Subtarget->hasNEON())) {
8103 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8104 }
8105 return Register();
8106}
8107
8108Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8109 if (RetVT.SimpleTy != MVT::v2i64)
8110 return Register();
8111 if ((Subtarget->hasNEON())) {
8112 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8113 }
8114 return Register();
8115}
8116
8117Register fastEmit_ARMISD_VSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8118 switch (VT.SimpleTy) {
8119 case MVT::v8i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8120 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8121 case MVT::v4i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8122 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8123 case MVT::v2i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8124 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8125 case MVT::v1i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8126 case MVT::v2i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8127 default: return Register();
8128 }
8129}
8130
8131// FastEmit functions for ARMISD::VSHRuIMM.
8132
8133Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8134 if (RetVT.SimpleTy != MVT::v8i8)
8135 return Register();
8136 if ((Subtarget->hasNEON())) {
8137 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8138 }
8139 return Register();
8140}
8141
8142Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8143 if (RetVT.SimpleTy != MVT::v16i8)
8144 return Register();
8145 if ((Subtarget->hasNEON())) {
8146 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8147 }
8148 return Register();
8149}
8150
8151Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8152 if (RetVT.SimpleTy != MVT::v4i16)
8153 return Register();
8154 if ((Subtarget->hasNEON())) {
8155 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8156 }
8157 return Register();
8158}
8159
8160Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8161 if (RetVT.SimpleTy != MVT::v8i16)
8162 return Register();
8163 if ((Subtarget->hasNEON())) {
8164 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8165 }
8166 return Register();
8167}
8168
8169Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8170 if (RetVT.SimpleTy != MVT::v2i32)
8171 return Register();
8172 if ((Subtarget->hasNEON())) {
8173 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8174 }
8175 return Register();
8176}
8177
8178Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8179 if (RetVT.SimpleTy != MVT::v4i32)
8180 return Register();
8181 if ((Subtarget->hasNEON())) {
8182 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8183 }
8184 return Register();
8185}
8186
8187Register fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8188 if (RetVT.SimpleTy != MVT::v1i64)
8189 return Register();
8190 if ((Subtarget->hasNEON())) {
8191 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8192 }
8193 return Register();
8194}
8195
8196Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8197 if (RetVT.SimpleTy != MVT::v2i64)
8198 return Register();
8199 if ((Subtarget->hasNEON())) {
8200 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8201 }
8202 return Register();
8203}
8204
8205Register fastEmit_ARMISD_VSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8206 switch (VT.SimpleTy) {
8207 case MVT::v8i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8208 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8209 case MVT::v4i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8210 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8211 case MVT::v2i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8212 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8213 case MVT::v1i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8214 case MVT::v2i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8215 default: return Register();
8216 }
8217}
8218
8219// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
8220
8221Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8222 if (RetVT.SimpleTy != MVT::i32)
8223 return Register();
8224 if ((Subtarget->hasFPRegs()) && (!Subtarget->hasSlowVGETLNi32())) {
8225 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNi32, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8226 }
8227 return Register();
8228}
8229
8230Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8231 switch (VT.SimpleTy) {
8232 case MVT::v2i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(RetVT, Op0, imm1);
8233 default: return Register();
8234 }
8235}
8236
8237// FastEmit functions for ISD::SHL.
8238
8239Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8240 if (RetVT.SimpleTy != MVT::i32)
8241 return Register();
8242 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8243 return fastEmitInst_ri(MachineInstOpcode: ARM::tLSLri, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8244 }
8245 return Register();
8246}
8247
8248Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8249 switch (VT.SimpleTy) {
8250 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
8251 default: return Register();
8252 }
8253}
8254
8255// Top-level FastEmit function.
8256
8257Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
8258 if (VT == MVT::i32 && Predicate_mod_imm(Imm: imm1))
8259 if (Register Reg = fastEmit_ri_Predicate_mod_imm(VT, RetVT, Opcode, Op0, imm1))
8260 return Reg;
8261
8262 if (VT == MVT::i32 && Predicate_imm0_7(Imm: imm1))
8263 if (Register Reg = fastEmit_ri_Predicate_imm0_7(VT, RetVT, Opcode, Op0, imm1))
8264 return Reg;
8265
8266 if (VT == MVT::i32 && Predicate_imm0_255_expr(Imm: imm1))
8267 if (Register Reg = fastEmit_ri_Predicate_imm0_255_expr(VT, RetVT, Opcode, Op0, imm1))
8268 return Reg;
8269
8270 if (VT == MVT::i32 && Predicate_t2_so_imm(Imm: imm1))
8271 if (Register Reg = fastEmit_ri_Predicate_t2_so_imm(VT, RetVT, Opcode, Op0, imm1))
8272 return Reg;
8273
8274 if (VT == MVT::i32 && Predicate_imm0_4095(Imm: imm1))
8275 if (Register Reg = fastEmit_ri_Predicate_imm0_4095(VT, RetVT, Opcode, Op0, imm1))
8276 return Reg;
8277
8278 if (VT == MVT::i32 && Predicate_imm1_31(Imm: imm1))
8279 if (Register Reg = fastEmit_ri_Predicate_imm1_31(VT, RetVT, Opcode, Op0, imm1))
8280 return Reg;
8281
8282 if (VT == MVT::i32 && Predicate_shr_imm8(Imm: imm1))
8283 if (Register Reg = fastEmit_ri_Predicate_shr_imm8(VT, RetVT, Opcode, Op0, imm1))
8284 return Reg;
8285
8286 if (VT == MVT::i32 && Predicate_shr_imm16(Imm: imm1))
8287 if (Register Reg = fastEmit_ri_Predicate_shr_imm16(VT, RetVT, Opcode, Op0, imm1))
8288 return Reg;
8289
8290 if (VT == MVT::i32 && Predicate_shr_imm32(Imm: imm1))
8291 if (Register Reg = fastEmit_ri_Predicate_shr_imm32(VT, RetVT, Opcode, Op0, imm1))
8292 return Reg;
8293
8294 if (VT == MVT::i32 && Predicate_VectorIndex32(Imm: imm1))
8295 if (Register Reg = fastEmit_ri_Predicate_VectorIndex32(VT, RetVT, Opcode, Op0, imm1))
8296 return Reg;
8297
8298 if (VT == MVT::i32 && Predicate_imm0_31(Imm: imm1))
8299 if (Register Reg = fastEmit_ri_Predicate_imm0_31(VT, RetVT, Opcode, Op0, imm1))
8300 return Reg;
8301
8302 if (VT == MVT::i32 && Predicate_imm0_15(Imm: imm1))
8303 if (Register Reg = fastEmit_ri_Predicate_imm0_15(VT, RetVT, Opcode, Op0, imm1))
8304 return Reg;
8305
8306 switch (Opcode) {
8307 case ARMISD::PIC_ADD: return fastEmit_ARMISD_PIC_ADD_ri(VT, RetVT, Op0, imm1);
8308 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri(VT, RetVT, Op0, imm1);
8309 case ARMISD::VGETLANEs: return fastEmit_ARMISD_VGETLANEs_ri(VT, RetVT, Op0, imm1);
8310 case ARMISD::VGETLANEu: return fastEmit_ARMISD_VGETLANEu_ri(VT, RetVT, Op0, imm1);
8311 case ARMISD::VQSHLsIMM: return fastEmit_ARMISD_VQSHLsIMM_ri(VT, RetVT, Op0, imm1);
8312 case ARMISD::VQSHLsuIMM: return fastEmit_ARMISD_VQSHLsuIMM_ri(VT, RetVT, Op0, imm1);
8313 case ARMISD::VQSHLuIMM: return fastEmit_ARMISD_VQSHLuIMM_ri(VT, RetVT, Op0, imm1);
8314 case ARMISD::VRSHRsIMM: return fastEmit_ARMISD_VRSHRsIMM_ri(VT, RetVT, Op0, imm1);
8315 case ARMISD::VRSHRuIMM: return fastEmit_ARMISD_VRSHRuIMM_ri(VT, RetVT, Op0, imm1);
8316 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri(VT, RetVT, Op0, imm1);
8317 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri(VT, RetVT, Op0, imm1);
8318 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri(VT, RetVT, Op0, imm1);
8319 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
8320 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
8321 default: return Register();
8322 }
8323}
8324
8325// FastEmit functions for ISD::ADD.
8326
8327Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8328 if (RetVT.SimpleTy != MVT::i32)
8329 return Register();
8330 if ((!Subtarget->isThumb())) {
8331 return fastEmitInst_ri(MachineInstOpcode: ARM::ADDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8332 }
8333 return Register();
8334}
8335
8336Register fastEmit_ISD_ADD_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8337 switch (VT.SimpleTy) {
8338 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8339 default: return Register();
8340 }
8341}
8342
8343// FastEmit functions for ISD::AND.
8344
8345Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8346 if (RetVT.SimpleTy != MVT::i32)
8347 return Register();
8348 if ((!Subtarget->isThumb())) {
8349 return fastEmitInst_ri(MachineInstOpcode: ARM::ANDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8350 }
8351 return Register();
8352}
8353
8354Register fastEmit_ISD_AND_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8355 switch (VT.SimpleTy) {
8356 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8357 default: return Register();
8358 }
8359}
8360
8361// FastEmit functions for ISD::OR.
8362
8363Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8364 if (RetVT.SimpleTy != MVT::i32)
8365 return Register();
8366 if ((!Subtarget->isThumb())) {
8367 return fastEmitInst_ri(MachineInstOpcode: ARM::ORRri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8368 }
8369 return Register();
8370}
8371
8372Register fastEmit_ISD_OR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8373 switch (VT.SimpleTy) {
8374 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8375 default: return Register();
8376 }
8377}
8378
8379// FastEmit functions for ISD::SUB.
8380
8381Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8382 if (RetVT.SimpleTy != MVT::i32)
8383 return Register();
8384 if ((!Subtarget->isThumb())) {
8385 return fastEmitInst_ri(MachineInstOpcode: ARM::SUBri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8386 }
8387 return Register();
8388}
8389
8390Register fastEmit_ISD_SUB_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8391 switch (VT.SimpleTy) {
8392 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8393 default: return Register();
8394 }
8395}
8396
8397// FastEmit functions for ISD::XOR.
8398
8399Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8400 if (RetVT.SimpleTy != MVT::i32)
8401 return Register();
8402 if ((!Subtarget->isThumb())) {
8403 return fastEmitInst_ri(MachineInstOpcode: ARM::EORri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8404 }
8405 return Register();
8406}
8407
8408Register fastEmit_ISD_XOR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8409 switch (VT.SimpleTy) {
8410 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8411 default: return Register();
8412 }
8413}
8414
8415// Top-level FastEmit function.
8416
8417Register fastEmit_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8418 switch (Opcode) {
8419 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8420 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8421 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8422 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8423 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8424 default: return Register();
8425 }
8426}
8427
8428// FastEmit functions for ARMISD::VSHLIMM.
8429
8430Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8431 if (RetVT.SimpleTy != MVT::v16i8)
8432 return Register();
8433 if ((Subtarget->hasMVEIntegerOps())) {
8434 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8435 }
8436 return Register();
8437}
8438
8439Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8440 switch (VT.SimpleTy) {
8441 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8442 default: return Register();
8443 }
8444}
8445
8446// FastEmit functions for ARMISD::VSHRsIMM.
8447
8448Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8449 if (RetVT.SimpleTy != MVT::v16i8)
8450 return Register();
8451 if ((Subtarget->hasMVEIntegerOps())) {
8452 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8453 }
8454 return Register();
8455}
8456
8457Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8458 switch (VT.SimpleTy) {
8459 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8460 default: return Register();
8461 }
8462}
8463
8464// FastEmit functions for ARMISD::VSHRuIMM.
8465
8466Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8467 if (RetVT.SimpleTy != MVT::v16i8)
8468 return Register();
8469 if ((Subtarget->hasMVEIntegerOps())) {
8470 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8471 }
8472 return Register();
8473}
8474
8475Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8476 switch (VT.SimpleTy) {
8477 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8478 default: return Register();
8479 }
8480}
8481
8482// FastEmit functions for ISD::ADD.
8483
8484Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8485 if (RetVT.SimpleTy != MVT::i32)
8486 return Register();
8487 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8488 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi3, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8489 }
8490 return Register();
8491}
8492
8493Register fastEmit_ISD_ADD_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8494 switch (VT.SimpleTy) {
8495 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8496 default: return Register();
8497 }
8498}
8499
8500// Top-level FastEmit function.
8501
8502Register fastEmit_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8503 switch (Opcode) {
8504 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8505 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8506 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8507 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8508 default: return Register();
8509 }
8510}
8511
8512// FastEmit functions for ISD::ADD.
8513
8514Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(MVT RetVT, Register Op0, uint64_t imm1) {
8515 if (RetVT.SimpleTy != MVT::i32)
8516 return Register();
8517 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8518 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8519 }
8520 return Register();
8521}
8522
8523Register fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8524 switch (VT.SimpleTy) {
8525 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(RetVT, Op0, imm1);
8526 default: return Register();
8527 }
8528}
8529
8530// Top-level FastEmit function.
8531
8532Register fastEmit_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8533 switch (Opcode) {
8534 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(VT, RetVT, Op0, imm1);
8535 default: return Register();
8536 }
8537}
8538
8539// FastEmit functions for ISD::ADD.
8540
8541Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8542 if (RetVT.SimpleTy != MVT::i32)
8543 return Register();
8544 if ((Subtarget->isThumb2())) {
8545 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8546 }
8547 return Register();
8548}
8549
8550Register fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8551 switch (VT.SimpleTy) {
8552 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8553 default: return Register();
8554 }
8555}
8556
8557// FastEmit functions for ISD::AND.
8558
8559Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8560 if (RetVT.SimpleTy != MVT::i32)
8561 return Register();
8562 if ((Subtarget->isThumb2())) {
8563 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ANDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8564 }
8565 return Register();
8566}
8567
8568Register fastEmit_ISD_AND_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8569 switch (VT.SimpleTy) {
8570 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8571 default: return Register();
8572 }
8573}
8574
8575// FastEmit functions for ISD::OR.
8576
8577Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8578 if (RetVT.SimpleTy != MVT::i32)
8579 return Register();
8580 if ((Subtarget->isThumb2())) {
8581 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ORRri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8582 }
8583 return Register();
8584}
8585
8586Register fastEmit_ISD_OR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8587 switch (VT.SimpleTy) {
8588 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8589 default: return Register();
8590 }
8591}
8592
8593// FastEmit functions for ISD::SUB.
8594
8595Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8596 if (RetVT.SimpleTy != MVT::i32)
8597 return Register();
8598 if ((Subtarget->isThumb2())) {
8599 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8600 }
8601 return Register();
8602}
8603
8604Register fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8605 switch (VT.SimpleTy) {
8606 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8607 default: return Register();
8608 }
8609}
8610
8611// FastEmit functions for ISD::XOR.
8612
8613Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8614 if (RetVT.SimpleTy != MVT::i32)
8615 return Register();
8616 if ((Subtarget->isThumb2())) {
8617 return fastEmitInst_ri(MachineInstOpcode: ARM::t2EORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8618 }
8619 return Register();
8620}
8621
8622Register fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8623 switch (VT.SimpleTy) {
8624 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8625 default: return Register();
8626 }
8627}
8628
8629// Top-level FastEmit function.
8630
8631Register fastEmit_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8632 switch (Opcode) {
8633 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8634 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8635 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8636 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8637 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8638 default: return Register();
8639 }
8640}
8641
8642// FastEmit functions for ISD::ADD.
8643
8644Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
8645 if (RetVT.SimpleTy != MVT::i32)
8646 return Register();
8647 if ((Subtarget->isThumb2())) {
8648 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8649 }
8650 return Register();
8651}
8652
8653Register fastEmit_ISD_ADD_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8654 switch (VT.SimpleTy) {
8655 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
8656 default: return Register();
8657 }
8658}
8659
8660// FastEmit functions for ISD::SUB.
8661
8662Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
8663 if (RetVT.SimpleTy != MVT::i32)
8664 return Register();
8665 if ((Subtarget->isThumb2())) {
8666 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8667 }
8668 return Register();
8669}
8670
8671Register fastEmit_ISD_SUB_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8672 switch (VT.SimpleTy) {
8673 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
8674 default: return Register();
8675 }
8676}
8677
8678// Top-level FastEmit function.
8679
8680Register fastEmit_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8681 switch (Opcode) {
8682 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
8683 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
8684 default: return Register();
8685 }
8686}
8687
8688// FastEmit functions for ISD::ROTR.
8689
8690Register fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
8691 if (RetVT.SimpleTy != MVT::i32)
8692 return Register();
8693 if ((Subtarget->isThumb2())) {
8694 return fastEmitInst_ri(MachineInstOpcode: ARM::t2RORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8695 }
8696 return Register();
8697}
8698
8699Register fastEmit_ISD_ROTR_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8700 switch (VT.SimpleTy) {
8701 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
8702 default: return Register();
8703 }
8704}
8705
8706// FastEmit functions for ISD::SHL.
8707
8708Register fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
8709 if (RetVT.SimpleTy != MVT::i32)
8710 return Register();
8711 if ((Subtarget->isThumb2())) {
8712 return fastEmitInst_ri(MachineInstOpcode: ARM::t2LSLri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8713 }
8714 return Register();
8715}
8716
8717Register fastEmit_ISD_SHL_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8718 switch (VT.SimpleTy) {
8719 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
8720 default: return Register();
8721 }
8722}
8723
8724// Top-level FastEmit function.
8725
8726Register fastEmit_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8727 switch (Opcode) {
8728 case ISD::ROTR: return fastEmit_ISD_ROTR_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
8729 case ISD::SHL: return fastEmit_ISD_SHL_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
8730 default: return Register();
8731 }
8732}
8733
8734// FastEmit functions for ARMISD::VQRSHRNsIMM.
8735
8736Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8737 if (RetVT.SimpleTy != MVT::v8i8)
8738 return Register();
8739 if ((Subtarget->hasNEON())) {
8740 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8741 }
8742 return Register();
8743}
8744
8745Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8746 switch (VT.SimpleTy) {
8747 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8748 default: return Register();
8749 }
8750}
8751
8752// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8753
8754Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8755 if (RetVT.SimpleTy != MVT::v8i8)
8756 return Register();
8757 if ((Subtarget->hasNEON())) {
8758 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8759 }
8760 return Register();
8761}
8762
8763Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8764 switch (VT.SimpleTy) {
8765 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8766 default: return Register();
8767 }
8768}
8769
8770// FastEmit functions for ARMISD::VQRSHRNuIMM.
8771
8772Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8773 if (RetVT.SimpleTy != MVT::v8i8)
8774 return Register();
8775 if ((Subtarget->hasNEON())) {
8776 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8777 }
8778 return Register();
8779}
8780
8781Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8782 switch (VT.SimpleTy) {
8783 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8784 default: return Register();
8785 }
8786}
8787
8788// FastEmit functions for ARMISD::VQSHRNsIMM.
8789
8790Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8791 if (RetVT.SimpleTy != MVT::v8i8)
8792 return Register();
8793 if ((Subtarget->hasNEON())) {
8794 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8795 }
8796 return Register();
8797}
8798
8799Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8800 switch (VT.SimpleTy) {
8801 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8802 default: return Register();
8803 }
8804}
8805
8806// FastEmit functions for ARMISD::VQSHRNsuIMM.
8807
8808Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8809 if (RetVT.SimpleTy != MVT::v8i8)
8810 return Register();
8811 if ((Subtarget->hasNEON())) {
8812 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8813 }
8814 return Register();
8815}
8816
8817Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8818 switch (VT.SimpleTy) {
8819 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8820 default: return Register();
8821 }
8822}
8823
8824// FastEmit functions for ARMISD::VQSHRNuIMM.
8825
8826Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8827 if (RetVT.SimpleTy != MVT::v8i8)
8828 return Register();
8829 if ((Subtarget->hasNEON())) {
8830 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8831 }
8832 return Register();
8833}
8834
8835Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8836 switch (VT.SimpleTy) {
8837 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8838 default: return Register();
8839 }
8840}
8841
8842// FastEmit functions for ARMISD::VRSHRNIMM.
8843
8844Register fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8845 if (RetVT.SimpleTy != MVT::v8i8)
8846 return Register();
8847 if ((Subtarget->hasNEON())) {
8848 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8849 }
8850 return Register();
8851}
8852
8853Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8854 switch (VT.SimpleTy) {
8855 case MVT::v8i16: return fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8856 default: return Register();
8857 }
8858}
8859
8860// Top-level FastEmit function.
8861
8862Register fastEmit_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8863 switch (Opcode) {
8864 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8865 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8866 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8867 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8868 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8869 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8870 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8871 default: return Register();
8872 }
8873}
8874
8875// FastEmit functions for ARMISD::VQRSHRNsIMM.
8876
8877Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8878 if (RetVT.SimpleTy != MVT::v4i16)
8879 return Register();
8880 if ((Subtarget->hasNEON())) {
8881 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8882 }
8883 return Register();
8884}
8885
8886Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8887 switch (VT.SimpleTy) {
8888 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8889 default: return Register();
8890 }
8891}
8892
8893// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8894
8895Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8896 if (RetVT.SimpleTy != MVT::v4i16)
8897 return Register();
8898 if ((Subtarget->hasNEON())) {
8899 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8900 }
8901 return Register();
8902}
8903
8904Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8905 switch (VT.SimpleTy) {
8906 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8907 default: return Register();
8908 }
8909}
8910
8911// FastEmit functions for ARMISD::VQRSHRNuIMM.
8912
8913Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8914 if (RetVT.SimpleTy != MVT::v4i16)
8915 return Register();
8916 if ((Subtarget->hasNEON())) {
8917 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8918 }
8919 return Register();
8920}
8921
8922Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8923 switch (VT.SimpleTy) {
8924 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8925 default: return Register();
8926 }
8927}
8928
8929// FastEmit functions for ARMISD::VQSHRNsIMM.
8930
8931Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8932 if (RetVT.SimpleTy != MVT::v4i16)
8933 return Register();
8934 if ((Subtarget->hasNEON())) {
8935 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8936 }
8937 return Register();
8938}
8939
8940Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8941 switch (VT.SimpleTy) {
8942 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8943 default: return Register();
8944 }
8945}
8946
8947// FastEmit functions for ARMISD::VQSHRNsuIMM.
8948
8949Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8950 if (RetVT.SimpleTy != MVT::v4i16)
8951 return Register();
8952 if ((Subtarget->hasNEON())) {
8953 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8954 }
8955 return Register();
8956}
8957
8958Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8959 switch (VT.SimpleTy) {
8960 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8961 default: return Register();
8962 }
8963}
8964
8965// FastEmit functions for ARMISD::VQSHRNuIMM.
8966
8967Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8968 if (RetVT.SimpleTy != MVT::v4i16)
8969 return Register();
8970 if ((Subtarget->hasNEON())) {
8971 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8972 }
8973 return Register();
8974}
8975
8976Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8977 switch (VT.SimpleTy) {
8978 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8979 default: return Register();
8980 }
8981}
8982
8983// FastEmit functions for ARMISD::VRSHRNIMM.
8984
8985Register fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8986 if (RetVT.SimpleTy != MVT::v4i16)
8987 return Register();
8988 if ((Subtarget->hasNEON())) {
8989 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8990 }
8991 return Register();
8992}
8993
8994Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8995 switch (VT.SimpleTy) {
8996 case MVT::v4i32: return fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8997 default: return Register();
8998 }
8999}
9000
9001// Top-level FastEmit function.
9002
9003Register fastEmit_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9004 switch (Opcode) {
9005 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9006 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9007 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9008 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9009 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9010 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9011 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9012 default: return Register();
9013 }
9014}
9015
9016// FastEmit functions for ARMISD::VQRSHRNsIMM.
9017
9018Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9019 if (RetVT.SimpleTy != MVT::v2i32)
9020 return Register();
9021 if ((Subtarget->hasNEON())) {
9022 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9023 }
9024 return Register();
9025}
9026
9027Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9028 switch (VT.SimpleTy) {
9029 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9030 default: return Register();
9031 }
9032}
9033
9034// FastEmit functions for ARMISD::VQRSHRNsuIMM.
9035
9036Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9037 if (RetVT.SimpleTy != MVT::v2i32)
9038 return Register();
9039 if ((Subtarget->hasNEON())) {
9040 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9041 }
9042 return Register();
9043}
9044
9045Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9046 switch (VT.SimpleTy) {
9047 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9048 default: return Register();
9049 }
9050}
9051
9052// FastEmit functions for ARMISD::VQRSHRNuIMM.
9053
9054Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9055 if (RetVT.SimpleTy != MVT::v2i32)
9056 return Register();
9057 if ((Subtarget->hasNEON())) {
9058 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9059 }
9060 return Register();
9061}
9062
9063Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9064 switch (VT.SimpleTy) {
9065 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9066 default: return Register();
9067 }
9068}
9069
9070// FastEmit functions for ARMISD::VQSHRNsIMM.
9071
9072Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9073 if (RetVT.SimpleTy != MVT::v2i32)
9074 return Register();
9075 if ((Subtarget->hasNEON())) {
9076 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9077 }
9078 return Register();
9079}
9080
9081Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9082 switch (VT.SimpleTy) {
9083 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9084 default: return Register();
9085 }
9086}
9087
9088// FastEmit functions for ARMISD::VQSHRNsuIMM.
9089
9090Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9091 if (RetVT.SimpleTy != MVT::v2i32)
9092 return Register();
9093 if ((Subtarget->hasNEON())) {
9094 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9095 }
9096 return Register();
9097}
9098
9099Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9100 switch (VT.SimpleTy) {
9101 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9102 default: return Register();
9103 }
9104}
9105
9106// FastEmit functions for ARMISD::VQSHRNuIMM.
9107
9108Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9109 if (RetVT.SimpleTy != MVT::v2i32)
9110 return Register();
9111 if ((Subtarget->hasNEON())) {
9112 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9113 }
9114 return Register();
9115}
9116
9117Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9118 switch (VT.SimpleTy) {
9119 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9120 default: return Register();
9121 }
9122}
9123
9124// FastEmit functions for ARMISD::VRSHRNIMM.
9125
9126Register fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9127 if (RetVT.SimpleTy != MVT::v2i32)
9128 return Register();
9129 if ((Subtarget->hasNEON())) {
9130 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9131 }
9132 return Register();
9133}
9134
9135Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9136 switch (VT.SimpleTy) {
9137 case MVT::v2i64: return fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9138 default: return Register();
9139 }
9140}
9141
9142// Top-level FastEmit function.
9143
9144Register fastEmit_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9145 switch (Opcode) {
9146 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9147 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9148 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9149 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9150 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9151 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9152 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9153 default: return Register();
9154 }
9155}
9156
9157// FastEmit functions for ARMISD::VDUPLANE.
9158
9159Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9160 if (RetVT.SimpleTy != MVT::v16i8)
9161 return Register();
9162 if ((Subtarget->hasNEON())) {
9163 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9164 }
9165 return Register();
9166}
9167
9168Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9169 if (RetVT.SimpleTy != MVT::v8i16)
9170 return Register();
9171 if ((Subtarget->hasNEON())) {
9172 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9173 }
9174 return Register();
9175}
9176
9177Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9178 if (RetVT.SimpleTy != MVT::v4i32)
9179 return Register();
9180 if ((Subtarget->hasNEON())) {
9181 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9182 }
9183 return Register();
9184}
9185
9186Register fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9187 switch (VT.SimpleTy) {
9188 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9189 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9190 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9191 default: return Register();
9192 }
9193}
9194
9195// Top-level FastEmit function.
9196
9197Register fastEmit_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9198 switch (Opcode) {
9199 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(VT, RetVT, Op0, imm1);
9200 default: return Register();
9201 }
9202}
9203
9204// FastEmit functions for ARMISD::VSHLIMM.
9205
9206Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9207 if (RetVT.SimpleTy != MVT::v4i32)
9208 return Register();
9209 if ((Subtarget->hasMVEIntegerOps())) {
9210 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9211 }
9212 return Register();
9213}
9214
9215Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9216 switch (VT.SimpleTy) {
9217 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9218 default: return Register();
9219 }
9220}
9221
9222// FastEmit functions for ARMISD::VSHRsIMM.
9223
9224Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9225 if (RetVT.SimpleTy != MVT::v4i32)
9226 return Register();
9227 if ((Subtarget->hasMVEIntegerOps())) {
9228 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9229 }
9230 return Register();
9231}
9232
9233Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9234 switch (VT.SimpleTy) {
9235 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9236 default: return Register();
9237 }
9238}
9239
9240// FastEmit functions for ARMISD::VSHRuIMM.
9241
9242Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9243 if (RetVT.SimpleTy != MVT::v4i32)
9244 return Register();
9245 if ((Subtarget->hasMVEIntegerOps())) {
9246 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9247 }
9248 return Register();
9249}
9250
9251Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9252 switch (VT.SimpleTy) {
9253 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9254 default: return Register();
9255 }
9256}
9257
9258// Top-level FastEmit function.
9259
9260Register fastEmit_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9261 switch (Opcode) {
9262 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9263 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9264 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9265 default: return Register();
9266 }
9267}
9268
9269// FastEmit functions for ARMISD::VSHLIMM.
9270
9271Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9272 if (RetVT.SimpleTy != MVT::v8i16)
9273 return Register();
9274 if ((Subtarget->hasMVEIntegerOps())) {
9275 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9276 }
9277 return Register();
9278}
9279
9280Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9281 switch (VT.SimpleTy) {
9282 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9283 default: return Register();
9284 }
9285}
9286
9287// FastEmit functions for ARMISD::VSHRsIMM.
9288
9289Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9290 if (RetVT.SimpleTy != MVT::v8i16)
9291 return Register();
9292 if ((Subtarget->hasMVEIntegerOps())) {
9293 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9294 }
9295 return Register();
9296}
9297
9298Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9299 switch (VT.SimpleTy) {
9300 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9301 default: return Register();
9302 }
9303}
9304
9305// FastEmit functions for ARMISD::VSHRuIMM.
9306
9307Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9308 if (RetVT.SimpleTy != MVT::v8i16)
9309 return Register();
9310 if ((Subtarget->hasMVEIntegerOps())) {
9311 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9312 }
9313 return Register();
9314}
9315
9316Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9317 switch (VT.SimpleTy) {
9318 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9319 default: return Register();
9320 }
9321}
9322
9323// Top-level FastEmit function.
9324
9325Register fastEmit_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9326 switch (Opcode) {
9327 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9328 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9329 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9330 default: return Register();
9331 }
9332}
9333
9334// FastEmit functions for ISD::Constant.
9335
9336Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
9337 if (RetVT.SimpleTy != MVT::i32)
9338 return Register();
9339 if ((Subtarget->isThumb()) && (Subtarget->useMovt())) {
9340 return fastEmitInst_i(MachineInstOpcode: ARM::t2MOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
9341 }
9342 if ((!Subtarget->useMovt()) && (Subtarget->genExecuteOnly()) && (Subtarget->isThumb1Only())) {
9343 return fastEmitInst_i(MachineInstOpcode: ARM::tMOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
9344 }
9345 return Register();
9346}
9347
9348Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
9349 switch (VT.SimpleTy) {
9350 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
9351 default: return Register();
9352 }
9353}
9354
9355// Top-level FastEmit function.
9356
9357Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
9358 switch (Opcode) {
9359 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
9360 default: return Register();
9361 }
9362}
9363
9364