1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the X86 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_AndMask64(int64_t Imm) {
12
13 return isMask_64(Value: Imm) && !isUInt<32>(x: Imm);
14
15}
16static bool Predicate_BTRMask64(int64_t Imm) {
17
18 return !isUInt<32>(x: Imm) && !isInt<32>(x: Imm) && isPowerOf2_64(Value: ~Imm);
19
20}
21static bool Predicate_BTCBTSMask64(int64_t Imm) {
22
23 return !isInt<32>(x: Imm) && isPowerOf2_64(Value: Imm);
24
25}
26static bool Predicate_i64immSExt32(int64_t Imm) {
27 return isInt<32>(x: Imm);
28}
29
30
31// FastEmit functions for ISD::ABS.
32
33unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
34 if (RetVT.SimpleTy != MVT::v16i8)
35 return 0;
36 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
37 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ128rr, RC: &X86::VR128XRegClass, Op0);
38 }
39 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
40 return fastEmitInst_r(MachineInstOpcode: X86::PABSBrr, RC: &X86::VR128RegClass, Op0);
41 }
42 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
43 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBrr, RC: &X86::VR128RegClass, Op0);
44 }
45 return 0;
46}
47
48unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0) {
49 if (RetVT.SimpleTy != MVT::v32i8)
50 return 0;
51 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
52 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ256rr, RC: &X86::VR256XRegClass, Op0);
53 }
54 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
55 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBYrr, RC: &X86::VR256RegClass, Op0);
56 }
57 return 0;
58}
59
60unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0) {
61 if (RetVT.SimpleTy != MVT::v64i8)
62 return 0;
63 if ((Subtarget->hasBWI())) {
64 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZrr, RC: &X86::VR512RegClass, Op0);
65 }
66 return 0;
67}
68
69unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
70 if (RetVT.SimpleTy != MVT::v8i16)
71 return 0;
72 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
73 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ128rr, RC: &X86::VR128XRegClass, Op0);
74 }
75 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
76 return fastEmitInst_r(MachineInstOpcode: X86::PABSWrr, RC: &X86::VR128RegClass, Op0);
77 }
78 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
79 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWrr, RC: &X86::VR128RegClass, Op0);
80 }
81 return 0;
82}
83
84unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
85 if (RetVT.SimpleTy != MVT::v16i16)
86 return 0;
87 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
88 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ256rr, RC: &X86::VR256XRegClass, Op0);
89 }
90 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
91 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWYrr, RC: &X86::VR256RegClass, Op0);
92 }
93 return 0;
94}
95
96unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
97 if (RetVT.SimpleTy != MVT::v32i16)
98 return 0;
99 if ((Subtarget->hasBWI())) {
100 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZrr, RC: &X86::VR512RegClass, Op0);
101 }
102 return 0;
103}
104
105unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
106 if (RetVT.SimpleTy != MVT::v4i32)
107 return 0;
108 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
109 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ128rr, RC: &X86::VR128XRegClass, Op0);
110 }
111 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
112 return fastEmitInst_r(MachineInstOpcode: X86::PABSDrr, RC: &X86::VR128RegClass, Op0);
113 }
114 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
115 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDrr, RC: &X86::VR128RegClass, Op0);
116 }
117 return 0;
118}
119
120unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
121 if (RetVT.SimpleTy != MVT::v8i32)
122 return 0;
123 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
124 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ256rr, RC: &X86::VR256XRegClass, Op0);
125 }
126 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
127 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDYrr, RC: &X86::VR256RegClass, Op0);
128 }
129 return 0;
130}
131
132unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
133 if (RetVT.SimpleTy != MVT::v16i32)
134 return 0;
135 if ((Subtarget->hasAVX512())) {
136 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZrr, RC: &X86::VR512RegClass, Op0);
137 }
138 return 0;
139}
140
141unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
142 if (RetVT.SimpleTy != MVT::v2i64)
143 return 0;
144 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
145 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ128rr, RC: &X86::VR128XRegClass, Op0);
146 }
147 return 0;
148}
149
150unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
151 if (RetVT.SimpleTy != MVT::v4i64)
152 return 0;
153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
154 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ256rr, RC: &X86::VR256XRegClass, Op0);
155 }
156 return 0;
157}
158
159unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
160 if (RetVT.SimpleTy != MVT::v8i64)
161 return 0;
162 if ((Subtarget->hasAVX512())) {
163 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZrr, RC: &X86::VR512RegClass, Op0);
164 }
165 return 0;
166}
167
168unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0) {
169 switch (VT.SimpleTy) {
170 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
171 case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0);
172 case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0);
173 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
174 case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0);
175 case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0);
176 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
177 case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0);
178 case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0);
179 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
180 case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0);
181 case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0);
182 default: return 0;
183 }
184}
185
186// FastEmit functions for ISD::ANY_EXTEND.
187
188unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0) {
189 if (RetVT.SimpleTy != MVT::i32)
190 return 0;
191 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
192}
193
194unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
195 switch (VT.SimpleTy) {
196 case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0);
197 default: return 0;
198 }
199}
200
201// FastEmit functions for ISD::BITCAST.
202
203unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0) {
204 if (RetVT.SimpleTy != MVT::f32)
205 return 0;
206 if ((Subtarget->hasAVX512())) {
207 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSZrr, RC: &X86::FR32XRegClass, Op0);
208 }
209 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
210 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
211 }
212 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
213 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
214 }
215 return 0;
216}
217
218unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0) {
219 if ((Subtarget->hasAVX512())) {
220 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDZrr, RC: &X86::FR64XRegClass, Op0);
221 }
222 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
223 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toSDrr, RC: &X86::FR64RegClass, Op0);
224 }
225 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
226 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDrr, RC: &X86::FR64RegClass, Op0);
227 }
228 return 0;
229}
230
231unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0) {
232 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
233 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64to64rr, RC: &X86::VR64RegClass, Op0);
234 }
235 return 0;
236}
237
238unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0) {
239switch (RetVT.SimpleTy) {
240 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0);
241 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0);
242 default: return 0;
243}
244}
245
246unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0) {
247 if (RetVT.SimpleTy != MVT::i32)
248 return 0;
249 if ((Subtarget->hasAVX512())) {
250 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIZrr, RC: &X86::GR32RegClass, Op0);
251 }
252 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
253 return fastEmitInst_r(MachineInstOpcode: X86::MOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
254 }
255 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
256 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
257 }
258 return 0;
259}
260
261unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0) {
262 if ((Subtarget->hasAVX512())) {
263 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64Zrr, RC: &X86::GR64RegClass, Op0);
264 }
265 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
266 return fastEmitInst_r(MachineInstOpcode: X86::MOVSDto64rr, RC: &X86::GR64RegClass, Op0);
267 }
268 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
269 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64rr, RC: &X86::GR64RegClass, Op0);
270 }
271 return 0;
272}
273
274unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0) {
275 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
276 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVFR642Qrr, RC: &X86::VR64RegClass, Op0);
277 }
278 return 0;
279}
280
281unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0) {
282switch (RetVT.SimpleTy) {
283 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0);
284 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0);
285 default: return 0;
286}
287}
288
289unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0) {
290 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
291 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64from64rr, RC: &X86::GR64RegClass, Op0);
292 }
293 return 0;
294}
295
296unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0) {
297 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
298 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2FR64rr, RC: &X86::FR64RegClass, Op0);
299 }
300 return 0;
301}
302
303unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0) {
304switch (RetVT.SimpleTy) {
305 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0);
306 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0);
307 default: return 0;
308}
309}
310
311unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0) {
312 switch (VT.SimpleTy) {
313 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
314 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0);
315 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
316 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
317 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0);
318 default: return 0;
319 }
320}
321
322// FastEmit functions for ISD::BRIND.
323
324unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0) {
325 if (RetVT.SimpleTy != MVT::isVoid)
326 return 0;
327 if ((!Subtarget->is64Bit())) {
328 return fastEmitInst_r(MachineInstOpcode: X86::JMP16r, RC: &X86::GR16RegClass, Op0);
329 }
330 return 0;
331}
332
333unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0) {
334 if (RetVT.SimpleTy != MVT::isVoid)
335 return 0;
336 if ((!Subtarget->is64Bit())) {
337 return fastEmitInst_r(MachineInstOpcode: X86::JMP32r, RC: &X86::GR32RegClass, Op0);
338 }
339 return 0;
340}
341
342unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0) {
343 if (RetVT.SimpleTy != MVT::isVoid)
344 return 0;
345 if ((Subtarget->is64Bit())) {
346 return fastEmitInst_r(MachineInstOpcode: X86::JMP64r, RC: &X86::GR64RegClass, Op0);
347 }
348 return 0;
349}
350
351unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0) {
352 switch (VT.SimpleTy) {
353 case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0);
354 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
355 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
356 default: return 0;
357 }
358}
359
360// FastEmit functions for ISD::BSWAP.
361
362unsigned fastEmit_ISD_BSWAP_MVT_i16_r(MVT RetVT, unsigned Op0) {
363 if (RetVT.SimpleTy != MVT::i16)
364 return 0;
365 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
366 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE16rr, RC: &X86::GR16RegClass, Op0);
367 }
368 return 0;
369}
370
371unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0) {
372 if (RetVT.SimpleTy != MVT::i32)
373 return 0;
374 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
375 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE32rr, RC: &X86::GR32RegClass, Op0);
376 }
377 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
378 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP32r, RC: &X86::GR32RegClass, Op0);
379 }
380 return 0;
381}
382
383unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0) {
384 if (RetVT.SimpleTy != MVT::i64)
385 return 0;
386 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
387 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE64rr, RC: &X86::GR64RegClass, Op0);
388 }
389 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
390 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP64r, RC: &X86::GR64RegClass, Op0);
391 }
392 return 0;
393}
394
395unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0) {
396 switch (VT.SimpleTy) {
397 case MVT::i16: return fastEmit_ISD_BSWAP_MVT_i16_r(RetVT, Op0);
398 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
399 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
400 default: return 0;
401 }
402}
403
404// FastEmit functions for ISD::CTLZ.
405
406unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0) {
407 if (RetVT.SimpleTy != MVT::i16)
408 return 0;
409 if ((Subtarget->hasLZCNT())) {
410 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT16rr, RC: &X86::GR16RegClass, Op0);
411 }
412 return 0;
413}
414
415unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0) {
416 if (RetVT.SimpleTy != MVT::i32)
417 return 0;
418 if ((Subtarget->hasLZCNT())) {
419 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT32rr, RC: &X86::GR32RegClass, Op0);
420 }
421 return 0;
422}
423
424unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0) {
425 if (RetVT.SimpleTy != MVT::i64)
426 return 0;
427 if ((Subtarget->hasLZCNT())) {
428 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT64rr, RC: &X86::GR64RegClass, Op0);
429 }
430 return 0;
431}
432
433unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
434 if (RetVT.SimpleTy != MVT::v4i32)
435 return 0;
436 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
437 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
438 }
439 return 0;
440}
441
442unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
443 if (RetVT.SimpleTy != MVT::v8i32)
444 return 0;
445 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
446 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
447 }
448 return 0;
449}
450
451unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
452 if (RetVT.SimpleTy != MVT::v16i32)
453 return 0;
454 if ((Subtarget->hasCDI())) {
455 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZrr, RC: &X86::VR512RegClass, Op0);
456 }
457 return 0;
458}
459
460unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
461 if (RetVT.SimpleTy != MVT::v2i64)
462 return 0;
463 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
464 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
465 }
466 return 0;
467}
468
469unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
470 if (RetVT.SimpleTy != MVT::v4i64)
471 return 0;
472 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
473 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
474 }
475 return 0;
476}
477
478unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
479 if (RetVT.SimpleTy != MVT::v8i64)
480 return 0;
481 if ((Subtarget->hasCDI())) {
482 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZrr, RC: &X86::VR512RegClass, Op0);
483 }
484 return 0;
485}
486
487unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0) {
488 switch (VT.SimpleTy) {
489 case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0);
490 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
491 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
492 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
493 case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0);
494 case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0);
495 case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0);
496 case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0);
497 case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0);
498 default: return 0;
499 }
500}
501
502// FastEmit functions for ISD::CTPOP.
503
504unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0) {
505 if (RetVT.SimpleTy != MVT::i16)
506 return 0;
507 if ((Subtarget->hasPOPCNT())) {
508 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT16rr, RC: &X86::GR16RegClass, Op0);
509 }
510 return 0;
511}
512
513unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0) {
514 if (RetVT.SimpleTy != MVT::i32)
515 return 0;
516 if ((Subtarget->hasPOPCNT())) {
517 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT32rr, RC: &X86::GR32RegClass, Op0);
518 }
519 return 0;
520}
521
522unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0) {
523 if (RetVT.SimpleTy != MVT::i64)
524 return 0;
525 if ((Subtarget->hasPOPCNT())) {
526 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT64rr, RC: &X86::GR64RegClass, Op0);
527 }
528 return 0;
529}
530
531unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
532 if (RetVT.SimpleTy != MVT::v16i8)
533 return 0;
534 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
535 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ128rr, RC: &X86::VR128XRegClass, Op0);
536 }
537 return 0;
538}
539
540unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0) {
541 if (RetVT.SimpleTy != MVT::v32i8)
542 return 0;
543 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
544 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ256rr, RC: &X86::VR256XRegClass, Op0);
545 }
546 return 0;
547}
548
549unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0) {
550 if (RetVT.SimpleTy != MVT::v64i8)
551 return 0;
552 if ((Subtarget->hasBITALG())) {
553 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZrr, RC: &X86::VR512RegClass, Op0);
554 }
555 return 0;
556}
557
558unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
559 if (RetVT.SimpleTy != MVT::v8i16)
560 return 0;
561 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
562 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ128rr, RC: &X86::VR128XRegClass, Op0);
563 }
564 return 0;
565}
566
567unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
568 if (RetVT.SimpleTy != MVT::v16i16)
569 return 0;
570 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
571 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ256rr, RC: &X86::VR256XRegClass, Op0);
572 }
573 return 0;
574}
575
576unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
577 if (RetVT.SimpleTy != MVT::v32i16)
578 return 0;
579 if ((Subtarget->hasBITALG())) {
580 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZrr, RC: &X86::VR512RegClass, Op0);
581 }
582 return 0;
583}
584
585unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
586 if (RetVT.SimpleTy != MVT::v4i32)
587 return 0;
588 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
589 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
590 }
591 return 0;
592}
593
594unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
595 if (RetVT.SimpleTy != MVT::v8i32)
596 return 0;
597 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
598 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
599 }
600 return 0;
601}
602
603unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
604 if (RetVT.SimpleTy != MVT::v16i32)
605 return 0;
606 if ((Subtarget->hasVPOPCNTDQ())) {
607 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZrr, RC: &X86::VR512RegClass, Op0);
608 }
609 return 0;
610}
611
612unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
613 if (RetVT.SimpleTy != MVT::v2i64)
614 return 0;
615 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
616 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
617 }
618 return 0;
619}
620
621unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
622 if (RetVT.SimpleTy != MVT::v4i64)
623 return 0;
624 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
625 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
626 }
627 return 0;
628}
629
630unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
631 if (RetVT.SimpleTy != MVT::v8i64)
632 return 0;
633 if ((Subtarget->hasVPOPCNTDQ())) {
634 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZrr, RC: &X86::VR512RegClass, Op0);
635 }
636 return 0;
637}
638
639unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0) {
640 switch (VT.SimpleTy) {
641 case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0);
642 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
643 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
644 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
645 case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0);
646 case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0);
647 case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0);
648 case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0);
649 case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0);
650 case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0);
651 case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0);
652 case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0);
653 case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0);
654 case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0);
655 case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0);
656 default: return 0;
657 }
658}
659
660// FastEmit functions for ISD::CTTZ.
661
662unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0) {
663 if (RetVT.SimpleTy != MVT::i16)
664 return 0;
665 if ((Subtarget->hasBMI())) {
666 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT16rr, RC: &X86::GR16RegClass, Op0);
667 }
668 return 0;
669}
670
671unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0) {
672 if (RetVT.SimpleTy != MVT::i32)
673 return 0;
674 if ((Subtarget->hasBMI())) {
675 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT32rr, RC: &X86::GR32RegClass, Op0);
676 }
677 return 0;
678}
679
680unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0) {
681 if (RetVT.SimpleTy != MVT::i64)
682 return 0;
683 if ((Subtarget->hasBMI())) {
684 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT64rr, RC: &X86::GR64RegClass, Op0);
685 }
686 return 0;
687}
688
689unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0) {
690 switch (VT.SimpleTy) {
691 case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0);
692 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
693 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
694 default: return 0;
695 }
696}
697
698// FastEmit functions for ISD::CTTZ_ZERO_UNDEF.
699
700unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0) {
701 if (RetVT.SimpleTy != MVT::i16)
702 return 0;
703 return fastEmitInst_r(MachineInstOpcode: X86::BSF16rr, RC: &X86::GR16RegClass, Op0);
704}
705
706unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0) {
707 if (RetVT.SimpleTy != MVT::i32)
708 return 0;
709 return fastEmitInst_r(MachineInstOpcode: X86::BSF32rr, RC: &X86::GR32RegClass, Op0);
710}
711
712unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0) {
713 if (RetVT.SimpleTy != MVT::i64)
714 return 0;
715 return fastEmitInst_r(MachineInstOpcode: X86::BSF64rr, RC: &X86::GR64RegClass, Op0);
716}
717
718unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0) {
719 switch (VT.SimpleTy) {
720 case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0);
721 case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0);
722 case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0);
723 default: return 0;
724 }
725}
726
727// FastEmit functions for ISD::FABS.
728
729unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0) {
730 if (RetVT.SimpleTy != MVT::f32)
731 return 0;
732 if ((!Subtarget->hasSSE1())) {
733 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp32, RC: &X86::RFP32RegClass, Op0);
734 }
735 return 0;
736}
737
738unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0) {
739 if (RetVT.SimpleTy != MVT::f64)
740 return 0;
741 if ((!Subtarget->hasSSE2())) {
742 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp64, RC: &X86::RFP64RegClass, Op0);
743 }
744 return 0;
745}
746
747unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0) {
748 if (RetVT.SimpleTy != MVT::f80)
749 return 0;
750 if ((Subtarget->hasX87())) {
751 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp80, RC: &X86::RFP80RegClass, Op0);
752 }
753 return 0;
754}
755
756unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0) {
757 switch (VT.SimpleTy) {
758 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
759 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
760 case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0);
761 default: return 0;
762 }
763}
764
765// FastEmit functions for ISD::FNEG.
766
767unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0) {
768 if (RetVT.SimpleTy != MVT::f32)
769 return 0;
770 if ((!Subtarget->hasSSE1())) {
771 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp32, RC: &X86::RFP32RegClass, Op0);
772 }
773 return 0;
774}
775
776unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0) {
777 if (RetVT.SimpleTy != MVT::f64)
778 return 0;
779 if ((!Subtarget->hasSSE2())) {
780 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp64, RC: &X86::RFP64RegClass, Op0);
781 }
782 return 0;
783}
784
785unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0) {
786 if (RetVT.SimpleTy != MVT::f80)
787 return 0;
788 if ((Subtarget->hasX87())) {
789 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp80, RC: &X86::RFP80RegClass, Op0);
790 }
791 return 0;
792}
793
794unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0) {
795 switch (VT.SimpleTy) {
796 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
797 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
798 case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0);
799 default: return 0;
800 }
801}
802
803// FastEmit functions for ISD::FP_EXTEND.
804
805unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) {
806 if (RetVT.SimpleTy != MVT::f64)
807 return 0;
808 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
809 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
810 }
811 return 0;
812}
813
814unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(unsigned Op0) {
815 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
816 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
817 }
818 return 0;
819}
820
821unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(unsigned Op0) {
822 if ((Subtarget->hasFP16())) {
823 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
824 }
825 return 0;
826}
827
828unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
829switch (RetVT.SimpleTy) {
830 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
831 case MVT::v8f64: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
832 default: return 0;
833}
834}
835
836unsigned fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
837 if (RetVT.SimpleTy != MVT::v16f32)
838 return 0;
839 if ((Subtarget->hasFP16())) {
840 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
841 }
842 return 0;
843}
844
845unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
846 if (RetVT.SimpleTy != MVT::v4f64)
847 return 0;
848 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
849 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
850 }
851 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
852 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
853 }
854 return 0;
855}
856
857unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
858 if (RetVT.SimpleTy != MVT::v8f64)
859 return 0;
860 if ((Subtarget->hasAVX512())) {
861 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
862 }
863 return 0;
864}
865
866unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
867 switch (VT.SimpleTy) {
868 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
869 case MVT::v8f16: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
870 case MVT::v16f16: return fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
871 case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
872 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
873 default: return 0;
874 }
875}
876
877// FastEmit functions for ISD::FP_ROUND.
878
879unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
880 if (RetVT.SimpleTy != MVT::f32)
881 return 0;
882 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
883 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
884 }
885 return 0;
886}
887
888unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
889 switch (VT.SimpleTy) {
890 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
891 default: return 0;
892 }
893}
894
895// FastEmit functions for ISD::FP_TO_SINT.
896
897unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0) {
898 if ((Subtarget->hasFP16())) {
899 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
900 }
901 return 0;
902}
903
904unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0) {
905 if ((Subtarget->hasFP16())) {
906 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
907 }
908 return 0;
909}
910
911unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
912switch (RetVT.SimpleTy) {
913 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
914 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
915 default: return 0;
916}
917}
918
919unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0) {
920 if ((Subtarget->hasAVX512())) {
921 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
922 }
923 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
924 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
925 }
926 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
927 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
928 }
929 return 0;
930}
931
932unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0) {
933 if ((Subtarget->hasAVX512())) {
934 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
935 }
936 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
937 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
938 }
939 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
940 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
941 }
942 return 0;
943}
944
945unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
946switch (RetVT.SimpleTy) {
947 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
948 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
949 default: return 0;
950}
951}
952
953unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0) {
954 if ((Subtarget->hasAVX512())) {
955 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
956 }
957 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
958 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
959 }
960 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
961 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
962 }
963 return 0;
964}
965
966unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0) {
967 if ((Subtarget->hasAVX512())) {
968 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
969 }
970 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
971 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
972 }
973 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
974 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
975 }
976 return 0;
977}
978
979unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
980switch (RetVT.SimpleTy) {
981 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
982 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
983 default: return 0;
984}
985}
986
987unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
988 if (RetVT.SimpleTy != MVT::v4i32)
989 return 0;
990 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
991 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
992 }
993 return 0;
994}
995
996unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) {
997 switch (VT.SimpleTy) {
998 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
999 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
1000 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
1001 case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
1002 default: return 0;
1003 }
1004}
1005
1006// FastEmit functions for ISD::FP_TO_UINT.
1007
1008unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0) {
1009 if ((Subtarget->hasFP16())) {
1010 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
1011 }
1012 return 0;
1013}
1014
1015unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0) {
1016 if ((Subtarget->hasFP16())) {
1017 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1018 }
1019 return 0;
1020}
1021
1022unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
1023switch (RetVT.SimpleTy) {
1024 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
1025 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
1026 default: return 0;
1027}
1028}
1029
1030unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0) {
1031 if ((Subtarget->hasAVX512())) {
1032 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
1033 }
1034 return 0;
1035}
1036
1037unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0) {
1038 if ((Subtarget->hasAVX512())) {
1039 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1040 }
1041 return 0;
1042}
1043
1044unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
1045switch (RetVT.SimpleTy) {
1046 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
1047 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
1048 default: return 0;
1049}
1050}
1051
1052unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0) {
1053 if ((Subtarget->hasAVX512())) {
1054 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
1055 }
1056 return 0;
1057}
1058
1059unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0) {
1060 if ((Subtarget->hasAVX512())) {
1061 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1062 }
1063 return 0;
1064}
1065
1066unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
1067switch (RetVT.SimpleTy) {
1068 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
1069 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
1070 default: return 0;
1071}
1072}
1073
1074unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) {
1075 switch (VT.SimpleTy) {
1076 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
1077 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
1078 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
1079 default: return 0;
1080 }
1081}
1082
1083// FastEmit functions for ISD::FSQRT.
1084
1085unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) {
1086 if (RetVT.SimpleTy != MVT::f32)
1087 return 0;
1088 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1089 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
1090 }
1091 if ((!Subtarget->hasSSE1())) {
1092 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
1093 }
1094 return 0;
1095}
1096
1097unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) {
1098 if (RetVT.SimpleTy != MVT::f64)
1099 return 0;
1100 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1101 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
1102 }
1103 if ((!Subtarget->hasSSE2())) {
1104 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
1105 }
1106 return 0;
1107}
1108
1109unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0) {
1110 if (RetVT.SimpleTy != MVT::f80)
1111 return 0;
1112 if ((Subtarget->hasX87())) {
1113 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
1114 }
1115 return 0;
1116}
1117
1118unsigned fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
1119 if (RetVT.SimpleTy != MVT::v8f16)
1120 return 0;
1121 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1122 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
1123 }
1124 return 0;
1125}
1126
1127unsigned fastEmit_ISD_FSQRT_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
1128 if (RetVT.SimpleTy != MVT::v16f16)
1129 return 0;
1130 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1131 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
1132 }
1133 return 0;
1134}
1135
1136unsigned fastEmit_ISD_FSQRT_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
1137 if (RetVT.SimpleTy != MVT::v32f16)
1138 return 0;
1139 if ((Subtarget->hasFP16())) {
1140 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
1141 }
1142 return 0;
1143}
1144
1145unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1146 if (RetVT.SimpleTy != MVT::v4f32)
1147 return 0;
1148 if ((Subtarget->hasVLX())) {
1149 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
1150 }
1151 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1152 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
1153 }
1154 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1155 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
1156 }
1157 return 0;
1158}
1159
1160unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
1161 if (RetVT.SimpleTy != MVT::v8f32)
1162 return 0;
1163 if ((Subtarget->hasVLX())) {
1164 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
1165 }
1166 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1167 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
1168 }
1169 return 0;
1170}
1171
1172unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
1173 if (RetVT.SimpleTy != MVT::v16f32)
1174 return 0;
1175 if ((Subtarget->hasAVX512())) {
1176 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
1177 }
1178 return 0;
1179}
1180
1181unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1182 if (RetVT.SimpleTy != MVT::v2f64)
1183 return 0;
1184 if ((Subtarget->hasVLX())) {
1185 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
1186 }
1187 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1188 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
1189 }
1190 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1191 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
1192 }
1193 return 0;
1194}
1195
1196unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
1197 if (RetVT.SimpleTy != MVT::v4f64)
1198 return 0;
1199 if ((Subtarget->hasVLX())) {
1200 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
1201 }
1202 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1203 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
1204 }
1205 return 0;
1206}
1207
1208unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
1209 if (RetVT.SimpleTy != MVT::v8f64)
1210 return 0;
1211 if ((Subtarget->hasAVX512())) {
1212 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
1213 }
1214 return 0;
1215}
1216
1217unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) {
1218 switch (VT.SimpleTy) {
1219 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
1220 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
1221 case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0);
1222 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
1223 case MVT::v16f16: return fastEmit_ISD_FSQRT_MVT_v16f16_r(RetVT, Op0);
1224 case MVT::v32f16: return fastEmit_ISD_FSQRT_MVT_v32f16_r(RetVT, Op0);
1225 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
1226 case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0);
1227 case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0);
1228 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
1229 case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0);
1230 case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0);
1231 default: return 0;
1232 }
1233}
1234
1235// FastEmit functions for ISD::LLRINT.
1236
1237unsigned fastEmit_ISD_LLRINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
1238 if (RetVT.SimpleTy != MVT::i64)
1239 return 0;
1240 if ((Subtarget->hasAVX512())) {
1241 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1242 }
1243 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1244 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1245 }
1246 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1247 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1248 }
1249 return 0;
1250}
1251
1252unsigned fastEmit_ISD_LLRINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
1253 if (RetVT.SimpleTy != MVT::i64)
1254 return 0;
1255 if ((Subtarget->hasAVX512())) {
1256 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1257 }
1258 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1259 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1260 }
1261 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1262 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1263 }
1264 return 0;
1265}
1266
1267unsigned fastEmit_ISD_LLRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1268 if (RetVT.SimpleTy != MVT::v4i64)
1269 return 0;
1270 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1271 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1272 }
1273 return 0;
1274}
1275
1276unsigned fastEmit_ISD_LLRINT_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
1277 if (RetVT.SimpleTy != MVT::v8i64)
1278 return 0;
1279 if ((Subtarget->hasDQI())) {
1280 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1281 }
1282 return 0;
1283}
1284
1285unsigned fastEmit_ISD_LLRINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1286 if (RetVT.SimpleTy != MVT::v2i64)
1287 return 0;
1288 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1289 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1290 }
1291 return 0;
1292}
1293
1294unsigned fastEmit_ISD_LLRINT_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
1295 if (RetVT.SimpleTy != MVT::v4i64)
1296 return 0;
1297 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1298 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1299 }
1300 return 0;
1301}
1302
1303unsigned fastEmit_ISD_LLRINT_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
1304 if (RetVT.SimpleTy != MVT::v8i64)
1305 return 0;
1306 if ((Subtarget->hasDQI())) {
1307 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1308 }
1309 return 0;
1310}
1311
1312unsigned fastEmit_ISD_LLRINT_r(MVT VT, MVT RetVT, unsigned Op0) {
1313 switch (VT.SimpleTy) {
1314 case MVT::f32: return fastEmit_ISD_LLRINT_MVT_f32_r(RetVT, Op0);
1315 case MVT::f64: return fastEmit_ISD_LLRINT_MVT_f64_r(RetVT, Op0);
1316 case MVT::v4f32: return fastEmit_ISD_LLRINT_MVT_v4f32_r(RetVT, Op0);
1317 case MVT::v8f32: return fastEmit_ISD_LLRINT_MVT_v8f32_r(RetVT, Op0);
1318 case MVT::v2f64: return fastEmit_ISD_LLRINT_MVT_v2f64_r(RetVT, Op0);
1319 case MVT::v4f64: return fastEmit_ISD_LLRINT_MVT_v4f64_r(RetVT, Op0);
1320 case MVT::v8f64: return fastEmit_ISD_LLRINT_MVT_v8f64_r(RetVT, Op0);
1321 default: return 0;
1322 }
1323}
1324
1325// FastEmit functions for ISD::LRINT.
1326
1327unsigned fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(unsigned Op0) {
1328 if ((Subtarget->hasAVX512())) {
1329 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
1330 }
1331 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1332 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1333 }
1334 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1335 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1336 }
1337 return 0;
1338}
1339
1340unsigned fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(unsigned Op0) {
1341 if ((Subtarget->hasAVX512())) {
1342 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1343 }
1344 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1345 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1346 }
1347 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1349 }
1350 return 0;
1351}
1352
1353unsigned fastEmit_ISD_LRINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
1354switch (RetVT.SimpleTy) {
1355 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Op0);
1356 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Op0);
1357 default: return 0;
1358}
1359}
1360
1361unsigned fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(unsigned Op0) {
1362 if ((Subtarget->hasAVX512())) {
1363 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
1364 }
1365 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1366 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1367 }
1368 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1369 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1370 }
1371 return 0;
1372}
1373
1374unsigned fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(unsigned Op0) {
1375 if ((Subtarget->hasAVX512())) {
1376 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1377 }
1378 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1379 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1380 }
1381 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1382 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1383 }
1384 return 0;
1385}
1386
1387unsigned fastEmit_ISD_LRINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
1388switch (RetVT.SimpleTy) {
1389 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Op0);
1390 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Op0);
1391 default: return 0;
1392}
1393}
1394
1395unsigned fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
1396 if ((Subtarget->hasVLX())) {
1397 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
1398 }
1399 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1400 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1401 }
1402 if ((Subtarget->hasAVX())) {
1403 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1404 }
1405 return 0;
1406}
1407
1408unsigned fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
1409 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1410 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1411 }
1412 return 0;
1413}
1414
1415unsigned fastEmit_ISD_LRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1416switch (RetVT.SimpleTy) {
1417 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Op0);
1418 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Op0);
1419 default: return 0;
1420}
1421}
1422
1423unsigned fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
1424 if ((Subtarget->hasVLX())) {
1425 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1426 }
1427 if ((Subtarget->hasAVX())) {
1428 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
1429 }
1430 return 0;
1431}
1432
1433unsigned fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
1434 if ((Subtarget->hasDQI())) {
1435 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1436 }
1437 return 0;
1438}
1439
1440unsigned fastEmit_ISD_LRINT_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
1441switch (RetVT.SimpleTy) {
1442 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Op0);
1443 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Op0);
1444 default: return 0;
1445}
1446}
1447
1448unsigned fastEmit_ISD_LRINT_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
1449 if (RetVT.SimpleTy != MVT::v16i32)
1450 return 0;
1451 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
1452}
1453
1454unsigned fastEmit_ISD_LRINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1455 if (RetVT.SimpleTy != MVT::v2i64)
1456 return 0;
1457 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1458 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1459 }
1460 return 0;
1461}
1462
1463unsigned fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
1464 if ((Subtarget->hasVLX())) {
1465 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
1466 }
1467 if ((Subtarget->hasAVX())) {
1468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
1469 }
1470 return 0;
1471}
1472
1473unsigned fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
1474 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1475 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1476 }
1477 return 0;
1478}
1479
1480unsigned fastEmit_ISD_LRINT_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
1481switch (RetVT.SimpleTy) {
1482 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Op0);
1483 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Op0);
1484 default: return 0;
1485}
1486}
1487
1488unsigned fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
1489 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
1490}
1491
1492unsigned fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
1493 if ((Subtarget->hasDQI())) {
1494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1495 }
1496 return 0;
1497}
1498
1499unsigned fastEmit_ISD_LRINT_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
1500switch (RetVT.SimpleTy) {
1501 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Op0);
1502 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Op0);
1503 default: return 0;
1504}
1505}
1506
1507unsigned fastEmit_ISD_LRINT_r(MVT VT, MVT RetVT, unsigned Op0) {
1508 switch (VT.SimpleTy) {
1509 case MVT::f32: return fastEmit_ISD_LRINT_MVT_f32_r(RetVT, Op0);
1510 case MVT::f64: return fastEmit_ISD_LRINT_MVT_f64_r(RetVT, Op0);
1511 case MVT::v4f32: return fastEmit_ISD_LRINT_MVT_v4f32_r(RetVT, Op0);
1512 case MVT::v8f32: return fastEmit_ISD_LRINT_MVT_v8f32_r(RetVT, Op0);
1513 case MVT::v16f32: return fastEmit_ISD_LRINT_MVT_v16f32_r(RetVT, Op0);
1514 case MVT::v2f64: return fastEmit_ISD_LRINT_MVT_v2f64_r(RetVT, Op0);
1515 case MVT::v4f64: return fastEmit_ISD_LRINT_MVT_v4f64_r(RetVT, Op0);
1516 case MVT::v8f64: return fastEmit_ISD_LRINT_MVT_v8f64_r(RetVT, Op0);
1517 default: return 0;
1518 }
1519}
1520
1521// FastEmit functions for ISD::SCALAR_TO_VECTOR.
1522
1523unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0) {
1524 if (RetVT.SimpleTy != MVT::v4i32)
1525 return 0;
1526 if ((Subtarget->hasAVX512())) {
1527 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIZrr, RC: &X86::VR128XRegClass, Op0);
1528 }
1529 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1530 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1531 }
1532 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1533 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1534 }
1535 return 0;
1536}
1537
1538unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0) {
1539 if (RetVT.SimpleTy != MVT::v2i64)
1540 return 0;
1541 if ((Subtarget->hasAVX512())) {
1542 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIZrr, RC: &X86::VR128XRegClass, Op0);
1543 }
1544 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1545 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1546 }
1547 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1548 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1549 }
1550 return 0;
1551}
1552
1553unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0) {
1554 switch (VT.SimpleTy) {
1555 case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0);
1556 case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0);
1557 default: return 0;
1558 }
1559}
1560
1561// FastEmit functions for ISD::SIGN_EXTEND.
1562
1563unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0) {
1564 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr8, RC: &X86::GR32RegClass, Op0);
1565}
1566
1567unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0) {
1568 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr8, RC: &X86::GR64RegClass, Op0);
1569}
1570
1571unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0) {
1572switch (RetVT.SimpleTy) {
1573 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0);
1574 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0);
1575 default: return 0;
1576}
1577}
1578
1579unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0) {
1580 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr16, RC: &X86::GR32RegClass, Op0);
1581}
1582
1583unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0) {
1584 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr16, RC: &X86::GR64RegClass, Op0);
1585}
1586
1587unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0) {
1588switch (RetVT.SimpleTy) {
1589 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0);
1590 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0);
1591 default: return 0;
1592}
1593}
1594
1595unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0) {
1596 if (RetVT.SimpleTy != MVT::i64)
1597 return 0;
1598 if ((Subtarget->is64Bit())) {
1599 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr32, RC: &X86::GR64RegClass, Op0);
1600 }
1601 return 0;
1602}
1603
1604unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0) {
1605 if (RetVT.SimpleTy != MVT::v2i64)
1606 return 0;
1607 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1608 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ128rr, RC: &X86::VR128XRegClass, Op0);
1609 }
1610 return 0;
1611}
1612
1613unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0) {
1614 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1615 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ128rr, RC: &X86::VR128XRegClass, Op0);
1616 }
1617 return 0;
1618}
1619
1620unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0) {
1621 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1622 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ256rr, RC: &X86::VR256XRegClass, Op0);
1623 }
1624 return 0;
1625}
1626
1627unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0) {
1628switch (RetVT.SimpleTy) {
1629 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0);
1630 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0);
1631 default: return 0;
1632}
1633}
1634
1635unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0) {
1636 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1637 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ128rr, RC: &X86::VR128XRegClass, Op0);
1638 }
1639 return 0;
1640}
1641
1642unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0) {
1643 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1644 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ256rr, RC: &X86::VR256XRegClass, Op0);
1645 }
1646 return 0;
1647}
1648
1649unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0) {
1650 if ((Subtarget->hasDQI())) {
1651 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZrr, RC: &X86::VR512RegClass, Op0);
1652 }
1653 return 0;
1654}
1655
1656unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0) {
1657switch (RetVT.SimpleTy) {
1658 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0);
1659 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0);
1660 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0);
1661 default: return 0;
1662}
1663}
1664
1665unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0) {
1666 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1667 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ128rr, RC: &X86::VR128XRegClass, Op0);
1668 }
1669 return 0;
1670}
1671
1672unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0) {
1673 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1674 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ256rr, RC: &X86::VR256XRegClass, Op0);
1675 }
1676 return 0;
1677}
1678
1679unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0) {
1680 if ((Subtarget->hasDQI())) {
1681 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZrr, RC: &X86::VR512RegClass, Op0);
1682 }
1683 return 0;
1684}
1685
1686unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0) {
1687switch (RetVT.SimpleTy) {
1688 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0);
1689 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0);
1690 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0);
1691 default: return 0;
1692}
1693}
1694
1695unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0) {
1696 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1697 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ256rr, RC: &X86::VR256XRegClass, Op0);
1698 }
1699 return 0;
1700}
1701
1702unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0) {
1703 if ((Subtarget->hasBWI())) {
1704 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZrr, RC: &X86::VR512RegClass, Op0);
1705 }
1706 return 0;
1707}
1708
1709unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0) {
1710switch (RetVT.SimpleTy) {
1711 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0);
1712 case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0);
1713 default: return 0;
1714}
1715}
1716
1717unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0) {
1718 if (RetVT.SimpleTy != MVT::v64i8)
1719 return 0;
1720 if ((Subtarget->hasBWI())) {
1721 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZrr, RC: &X86::VR512RegClass, Op0);
1722 }
1723 return 0;
1724}
1725
1726unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(unsigned Op0) {
1727 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1728 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWYrr, RC: &X86::VR256RegClass, Op0);
1729 }
1730 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1731 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
1732 }
1733 return 0;
1734}
1735
1736unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(unsigned Op0) {
1737 if ((Subtarget->hasAVX512())) {
1738 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZrr, RC: &X86::VR512RegClass, Op0);
1739 }
1740 return 0;
1741}
1742
1743unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
1744switch (RetVT.SimpleTy) {
1745 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
1746 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
1747 default: return 0;
1748}
1749}
1750
1751unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(MVT RetVT, unsigned Op0) {
1752 if (RetVT.SimpleTy != MVT::v32i16)
1753 return 0;
1754 if ((Subtarget->hasBWI())) {
1755 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZrr, RC: &X86::VR512RegClass, Op0);
1756 }
1757 return 0;
1758}
1759
1760unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(unsigned Op0) {
1761 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1762 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDYrr, RC: &X86::VR256RegClass, Op0);
1763 }
1764 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1765 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
1766 }
1767 return 0;
1768}
1769
1770unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(unsigned Op0) {
1771 if ((Subtarget->hasAVX512())) {
1772 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZrr, RC: &X86::VR512RegClass, Op0);
1773 }
1774 return 0;
1775}
1776
1777unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
1778switch (RetVT.SimpleTy) {
1779 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
1780 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
1781 default: return 0;
1782}
1783}
1784
1785unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
1786 if (RetVT.SimpleTy != MVT::v16i32)
1787 return 0;
1788 if ((Subtarget->hasAVX512())) {
1789 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZrr, RC: &X86::VR512RegClass, Op0);
1790 }
1791 return 0;
1792}
1793
1794unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
1795 if (RetVT.SimpleTy != MVT::v4i64)
1796 return 0;
1797 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1798 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQYrr, RC: &X86::VR256RegClass, Op0);
1799 }
1800 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1801 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
1802 }
1803 return 0;
1804}
1805
1806unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
1807 if (RetVT.SimpleTy != MVT::v8i64)
1808 return 0;
1809 if ((Subtarget->hasAVX512())) {
1810 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZrr, RC: &X86::VR512RegClass, Op0);
1811 }
1812 return 0;
1813}
1814
1815unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
1816 switch (VT.SimpleTy) {
1817 case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0);
1818 case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0);
1819 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0);
1820 case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0);
1821 case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0);
1822 case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0);
1823 case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0);
1824 case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0);
1825 case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0);
1826 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(RetVT, Op0);
1827 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(RetVT, Op0);
1828 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(RetVT, Op0);
1829 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(RetVT, Op0);
1830 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(RetVT, Op0);
1831 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(RetVT, Op0);
1832 default: return 0;
1833 }
1834}
1835
1836// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
1837
1838unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0) {
1839 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1840 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1841 }
1842 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1843 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1844 }
1845 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1846 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
1847 }
1848 return 0;
1849}
1850
1851unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0) {
1852 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1853 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
1854 }
1855 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1856 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
1857 }
1858 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1859 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
1860 }
1861 return 0;
1862}
1863
1864unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(unsigned Op0) {
1865 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1866 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDYrr, RC: &X86::VR256RegClass, Op0);
1867 }
1868 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1869 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
1870 }
1871 return 0;
1872}
1873
1874unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0) {
1875 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1876 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
1877 }
1878 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1879 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
1880 }
1881 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1882 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
1883 }
1884 return 0;
1885}
1886
1887unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(unsigned Op0) {
1888 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1889 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQYrr, RC: &X86::VR256RegClass, Op0);
1890 }
1891 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1892 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
1893 }
1894 return 0;
1895}
1896
1897unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(unsigned Op0) {
1898 if ((Subtarget->hasAVX512())) {
1899 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZrr, RC: &X86::VR512RegClass, Op0);
1900 }
1901 return 0;
1902}
1903
1904unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
1905switch (RetVT.SimpleTy) {
1906 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
1907 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
1908 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
1909 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
1910 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
1911 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
1912 default: return 0;
1913}
1914}
1915
1916unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0) {
1917 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1918 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
1919 }
1920 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1921 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
1922 }
1923 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1924 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
1925 }
1926 return 0;
1927}
1928
1929unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0) {
1930 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1931 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
1932 }
1933 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1934 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
1935 }
1936 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1937 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
1938 }
1939 return 0;
1940}
1941
1942unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(unsigned Op0) {
1943 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1944 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQYrr, RC: &X86::VR256RegClass, Op0);
1945 }
1946 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1947 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
1948 }
1949 return 0;
1950}
1951
1952unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
1953switch (RetVT.SimpleTy) {
1954 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
1955 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
1956 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
1957 default: return 0;
1958}
1959}
1960
1961unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
1962 if (RetVT.SimpleTy != MVT::v2i64)
1963 return 0;
1964 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1965 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
1966 }
1967 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1968 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
1969 }
1970 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1971 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
1972 }
1973 return 0;
1974}
1975
1976unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0) {
1977 switch (VT.SimpleTy) {
1978 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
1979 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
1980 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
1981 default: return 0;
1982 }
1983}
1984
1985// FastEmit functions for ISD::SINT_TO_FP.
1986
1987unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) {
1988 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1989 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
1990 }
1991 return 0;
1992}
1993
1994unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) {
1995 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1996 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
1997 }
1998 return 0;
1999}
2000
2001unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) {
2002switch (RetVT.SimpleTy) {
2003 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2004 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2005 default: return 0;
2006}
2007}
2008
2009unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) {
2010 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2011 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2012 }
2013 return 0;
2014}
2015
2016unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) {
2017 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2018 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2019 }
2020 return 0;
2021}
2022
2023unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) {
2024switch (RetVT.SimpleTy) {
2025 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2026 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2027 default: return 0;
2028}
2029}
2030
2031unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2032 if (RetVT.SimpleTy != MVT::v8f16)
2033 return 0;
2034 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2035 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2036 }
2037 return 0;
2038}
2039
2040unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
2041 if (RetVT.SimpleTy != MVT::v16f16)
2042 return 0;
2043 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2044 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2045 }
2046 return 0;
2047}
2048
2049unsigned fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
2050 if (RetVT.SimpleTy != MVT::v32f16)
2051 return 0;
2052 if ((Subtarget->hasFP16())) {
2053 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2054 }
2055 return 0;
2056}
2057
2058unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) {
2059 if ((Subtarget->hasVLX())) {
2060 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2061 }
2062 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2063 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2064 }
2065 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2066 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2067 }
2068 return 0;
2069}
2070
2071unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) {
2072 if ((Subtarget->hasVLX())) {
2073 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2074 }
2075 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2076 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2077 }
2078 return 0;
2079}
2080
2081unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2082switch (RetVT.SimpleTy) {
2083 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2084 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2085 default: return 0;
2086}
2087}
2088
2089unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) {
2090 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2091 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2092 }
2093 return 0;
2094}
2095
2096unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) {
2097 if ((Subtarget->hasVLX())) {
2098 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2099 }
2100 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2101 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2102 }
2103 return 0;
2104}
2105
2106unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) {
2107 if ((Subtarget->hasAVX512())) {
2108 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2109 }
2110 return 0;
2111}
2112
2113unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
2114switch (RetVT.SimpleTy) {
2115 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2116 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2117 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2118 default: return 0;
2119}
2120}
2121
2122unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) {
2123 if ((Subtarget->hasFP16())) {
2124 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2125 }
2126 return 0;
2127}
2128
2129unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) {
2130 if ((Subtarget->hasAVX512())) {
2131 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2132 }
2133 return 0;
2134}
2135
2136unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
2137switch (RetVT.SimpleTy) {
2138 case MVT::v16f16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2139 case MVT::v16f32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2140 default: return 0;
2141}
2142}
2143
2144unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
2145 if (RetVT.SimpleTy != MVT::v2f64)
2146 return 0;
2147 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2148 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2149 }
2150 return 0;
2151}
2152
2153unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) {
2154 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2155 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2156 }
2157 return 0;
2158}
2159
2160unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) {
2161 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2162 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2163 }
2164 return 0;
2165}
2166
2167unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
2168switch (RetVT.SimpleTy) {
2169 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2170 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2171 default: return 0;
2172}
2173}
2174
2175unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) {
2176 if ((Subtarget->hasFP16())) {
2177 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2178 }
2179 return 0;
2180}
2181
2182unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) {
2183 if ((Subtarget->hasDQI())) {
2184 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2185 }
2186 return 0;
2187}
2188
2189unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) {
2190 if ((Subtarget->hasDQI())) {
2191 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2192 }
2193 return 0;
2194}
2195
2196unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
2197switch (RetVT.SimpleTy) {
2198 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
2199 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
2200 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
2201 default: return 0;
2202}
2203}
2204
2205unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
2206 switch (VT.SimpleTy) {
2207 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
2208 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
2209 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2210 case MVT::v16i16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
2211 case MVT::v32i16: return fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
2212 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2213 case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
2214 case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
2215 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
2216 case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
2217 case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
2218 default: return 0;
2219 }
2220}
2221
2222// FastEmit functions for ISD::STRICT_FP_EXTEND.
2223
2224unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) {
2225 if (RetVT.SimpleTy != MVT::f64)
2226 return 0;
2227 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2228 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
2229 }
2230 return 0;
2231}
2232
2233unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(unsigned Op0) {
2234 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2235 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
2236 }
2237 return 0;
2238}
2239
2240unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(unsigned Op0) {
2241 if ((Subtarget->hasFP16())) {
2242 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
2243 }
2244 return 0;
2245}
2246
2247unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2248switch (RetVT.SimpleTy) {
2249 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
2250 case MVT::v8f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
2251 default: return 0;
2252}
2253}
2254
2255unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
2256 if (RetVT.SimpleTy != MVT::v16f32)
2257 return 0;
2258 if ((Subtarget->hasFP16())) {
2259 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
2260 }
2261 return 0;
2262}
2263
2264unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2265 if (RetVT.SimpleTy != MVT::v4f64)
2266 return 0;
2267 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2268 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2269 }
2270 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2271 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
2272 }
2273 return 0;
2274}
2275
2276unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
2277 if (RetVT.SimpleTy != MVT::v8f64)
2278 return 0;
2279 if ((Subtarget->hasAVX512())) {
2280 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
2281 }
2282 return 0;
2283}
2284
2285unsigned fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
2286 switch (VT.SimpleTy) {
2287 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2288 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
2289 case MVT::v16f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
2290 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
2291 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
2292 default: return 0;
2293 }
2294}
2295
2296// FastEmit functions for ISD::STRICT_FP_ROUND.
2297
2298unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
2299 if (RetVT.SimpleTy != MVT::f32)
2300 return 0;
2301 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2302 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
2303 }
2304 return 0;
2305}
2306
2307unsigned fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
2308 switch (VT.SimpleTy) {
2309 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
2310 default: return 0;
2311 }
2312}
2313
2314// FastEmit functions for ISD::STRICT_FP_TO_SINT.
2315
2316unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0) {
2317 if ((Subtarget->hasFP16())) {
2318 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
2319 }
2320 return 0;
2321}
2322
2323unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0) {
2324 if ((Subtarget->hasFP16())) {
2325 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2326 }
2327 return 0;
2328}
2329
2330unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
2331switch (RetVT.SimpleTy) {
2332 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
2333 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
2334 default: return 0;
2335}
2336}
2337
2338unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0) {
2339 if ((Subtarget->hasAVX512())) {
2340 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
2341 }
2342 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2343 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2344 }
2345 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2346 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2347 }
2348 return 0;
2349}
2350
2351unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0) {
2352 if ((Subtarget->hasAVX512())) {
2353 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2354 }
2355 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2356 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2357 }
2358 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2359 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2360 }
2361 return 0;
2362}
2363
2364unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
2365switch (RetVT.SimpleTy) {
2366 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
2367 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
2368 default: return 0;
2369}
2370}
2371
2372unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0) {
2373 if ((Subtarget->hasAVX512())) {
2374 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
2375 }
2376 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2377 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2378 }
2379 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2380 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2381 }
2382 return 0;
2383}
2384
2385unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0) {
2386 if ((Subtarget->hasAVX512())) {
2387 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2388 }
2389 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2390 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2391 }
2392 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2393 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2394 }
2395 return 0;
2396}
2397
2398unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
2399switch (RetVT.SimpleTy) {
2400 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
2401 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
2402 default: return 0;
2403}
2404}
2405
2406unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
2407 if (RetVT.SimpleTy != MVT::v4i32)
2408 return 0;
2409 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2410 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
2411 }
2412 return 0;
2413}
2414
2415unsigned fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) {
2416 switch (VT.SimpleTy) {
2417 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
2418 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
2419 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
2420 case MVT::v4f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
2421 default: return 0;
2422 }
2423}
2424
2425// FastEmit functions for ISD::STRICT_FP_TO_UINT.
2426
2427unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0) {
2428 if ((Subtarget->hasFP16())) {
2429 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
2430 }
2431 return 0;
2432}
2433
2434unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0) {
2435 if ((Subtarget->hasFP16())) {
2436 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2437 }
2438 return 0;
2439}
2440
2441unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
2442switch (RetVT.SimpleTy) {
2443 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
2444 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
2445 default: return 0;
2446}
2447}
2448
2449unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0) {
2450 if ((Subtarget->hasAVX512())) {
2451 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
2452 }
2453 return 0;
2454}
2455
2456unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0) {
2457 if ((Subtarget->hasAVX512())) {
2458 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2459 }
2460 return 0;
2461}
2462
2463unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
2464switch (RetVT.SimpleTy) {
2465 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
2466 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
2467 default: return 0;
2468}
2469}
2470
2471unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0) {
2472 if ((Subtarget->hasAVX512())) {
2473 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
2474 }
2475 return 0;
2476}
2477
2478unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0) {
2479 if ((Subtarget->hasAVX512())) {
2480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2481 }
2482 return 0;
2483}
2484
2485unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
2486switch (RetVT.SimpleTy) {
2487 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
2488 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
2489 default: return 0;
2490}
2491}
2492
2493unsigned fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) {
2494 switch (VT.SimpleTy) {
2495 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
2496 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
2497 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
2498 default: return 0;
2499 }
2500}
2501
2502// FastEmit functions for ISD::STRICT_FSQRT.
2503
2504unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) {
2505 if (RetVT.SimpleTy != MVT::f32)
2506 return 0;
2507 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2508 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
2509 }
2510 if ((!Subtarget->hasSSE1())) {
2511 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
2512 }
2513 return 0;
2514}
2515
2516unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) {
2517 if (RetVT.SimpleTy != MVT::f64)
2518 return 0;
2519 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2520 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
2521 }
2522 if ((!Subtarget->hasSSE2())) {
2523 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
2524 }
2525 return 0;
2526}
2527
2528unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0) {
2529 if (RetVT.SimpleTy != MVT::f80)
2530 return 0;
2531 if ((Subtarget->hasX87())) {
2532 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
2533 }
2534 return 0;
2535}
2536
2537unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2538 if (RetVT.SimpleTy != MVT::v8f16)
2539 return 0;
2540 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2541 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
2542 }
2543 return 0;
2544}
2545
2546unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
2547 if (RetVT.SimpleTy != MVT::v16f16)
2548 return 0;
2549 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2550 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
2551 }
2552 return 0;
2553}
2554
2555unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
2556 if (RetVT.SimpleTy != MVT::v32f16)
2557 return 0;
2558 if ((Subtarget->hasFP16())) {
2559 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
2560 }
2561 return 0;
2562}
2563
2564unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2565 if (RetVT.SimpleTy != MVT::v4f32)
2566 return 0;
2567 if ((Subtarget->hasVLX())) {
2568 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
2569 }
2570 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2571 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
2572 }
2573 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2574 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
2575 }
2576 return 0;
2577}
2578
2579unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
2580 if (RetVT.SimpleTy != MVT::v8f32)
2581 return 0;
2582 if ((Subtarget->hasVLX())) {
2583 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
2584 }
2585 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2586 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
2587 }
2588 return 0;
2589}
2590
2591unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
2592 if (RetVT.SimpleTy != MVT::v16f32)
2593 return 0;
2594 if ((Subtarget->hasAVX512())) {
2595 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
2596 }
2597 return 0;
2598}
2599
2600unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
2601 if (RetVT.SimpleTy != MVT::v2f64)
2602 return 0;
2603 if ((Subtarget->hasVLX())) {
2604 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
2605 }
2606 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2607 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
2608 }
2609 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2610 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
2611 }
2612 return 0;
2613}
2614
2615unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
2616 if (RetVT.SimpleTy != MVT::v4f64)
2617 return 0;
2618 if ((Subtarget->hasVLX())) {
2619 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
2620 }
2621 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2622 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
2623 }
2624 return 0;
2625}
2626
2627unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
2628 if (RetVT.SimpleTy != MVT::v8f64)
2629 return 0;
2630 if ((Subtarget->hasAVX512())) {
2631 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
2632 }
2633 return 0;
2634}
2635
2636unsigned fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) {
2637 switch (VT.SimpleTy) {
2638 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
2639 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
2640 case MVT::f80: return fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(RetVT, Op0);
2641 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
2642 case MVT::v16f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(RetVT, Op0);
2643 case MVT::v32f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(RetVT, Op0);
2644 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
2645 case MVT::v8f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(RetVT, Op0);
2646 case MVT::v16f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(RetVT, Op0);
2647 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
2648 case MVT::v4f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(RetVT, Op0);
2649 case MVT::v8f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(RetVT, Op0);
2650 default: return 0;
2651 }
2652}
2653
2654// FastEmit functions for ISD::STRICT_SINT_TO_FP.
2655
2656unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) {
2657 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2658 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2659 }
2660 return 0;
2661}
2662
2663unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) {
2664 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2665 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2666 }
2667 return 0;
2668}
2669
2670unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) {
2671switch (RetVT.SimpleTy) {
2672 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2673 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2674 default: return 0;
2675}
2676}
2677
2678unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) {
2679 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2680 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2681 }
2682 return 0;
2683}
2684
2685unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) {
2686 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2687 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2688 }
2689 return 0;
2690}
2691
2692unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) {
2693switch (RetVT.SimpleTy) {
2694 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2695 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2696 default: return 0;
2697}
2698}
2699
2700unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2701 if (RetVT.SimpleTy != MVT::v8f16)
2702 return 0;
2703 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2704 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2705 }
2706 return 0;
2707}
2708
2709unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
2710 if (RetVT.SimpleTy != MVT::v16f16)
2711 return 0;
2712 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2713 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2714 }
2715 return 0;
2716}
2717
2718unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
2719 if (RetVT.SimpleTy != MVT::v32f16)
2720 return 0;
2721 if ((Subtarget->hasFP16())) {
2722 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2723 }
2724 return 0;
2725}
2726
2727unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) {
2728 if ((Subtarget->hasVLX())) {
2729 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2730 }
2731 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2732 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2733 }
2734 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2735 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2736 }
2737 return 0;
2738}
2739
2740unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) {
2741 if ((Subtarget->hasVLX())) {
2742 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2743 }
2744 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2745 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2746 }
2747 return 0;
2748}
2749
2750unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2751switch (RetVT.SimpleTy) {
2752 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2753 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2754 default: return 0;
2755}
2756}
2757
2758unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) {
2759 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2760 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2761 }
2762 return 0;
2763}
2764
2765unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) {
2766 if ((Subtarget->hasVLX())) {
2767 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2768 }
2769 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2770 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2771 }
2772 return 0;
2773}
2774
2775unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) {
2776 if ((Subtarget->hasAVX512())) {
2777 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2778 }
2779 return 0;
2780}
2781
2782unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
2783switch (RetVT.SimpleTy) {
2784 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2785 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2786 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2787 default: return 0;
2788}
2789}
2790
2791unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) {
2792 if ((Subtarget->hasFP16())) {
2793 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2794 }
2795 return 0;
2796}
2797
2798unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) {
2799 if ((Subtarget->hasAVX512())) {
2800 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2801 }
2802 return 0;
2803}
2804
2805unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
2806switch (RetVT.SimpleTy) {
2807 case MVT::v16f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2808 case MVT::v16f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2809 default: return 0;
2810}
2811}
2812
2813unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
2814 if (RetVT.SimpleTy != MVT::v2f64)
2815 return 0;
2816 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2817 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2818 }
2819 return 0;
2820}
2821
2822unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) {
2823 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2824 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2825 }
2826 return 0;
2827}
2828
2829unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) {
2830 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2831 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2832 }
2833 return 0;
2834}
2835
2836unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
2837switch (RetVT.SimpleTy) {
2838 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2839 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2840 default: return 0;
2841}
2842}
2843
2844unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) {
2845 if ((Subtarget->hasFP16())) {
2846 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2847 }
2848 return 0;
2849}
2850
2851unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) {
2852 if ((Subtarget->hasDQI())) {
2853 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2854 }
2855 return 0;
2856}
2857
2858unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) {
2859 if ((Subtarget->hasDQI())) {
2860 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2861 }
2862 return 0;
2863}
2864
2865unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
2866switch (RetVT.SimpleTy) {
2867 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
2868 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
2869 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
2870 default: return 0;
2871}
2872}
2873
2874unsigned fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
2875 switch (VT.SimpleTy) {
2876 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
2877 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
2878 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2879 case MVT::v16i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
2880 case MVT::v32i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
2881 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2882 case MVT::v8i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
2883 case MVT::v16i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
2884 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
2885 case MVT::v4i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
2886 case MVT::v8i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
2887 default: return 0;
2888 }
2889}
2890
2891// FastEmit functions for ISD::STRICT_UINT_TO_FP.
2892
2893unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2894 if (RetVT.SimpleTy != MVT::v8f16)
2895 return 0;
2896 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2897 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2898 }
2899 return 0;
2900}
2901
2902unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
2903 if (RetVT.SimpleTy != MVT::v16f16)
2904 return 0;
2905 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2906 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2907 }
2908 return 0;
2909}
2910
2911unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
2912 if (RetVT.SimpleTy != MVT::v32f16)
2913 return 0;
2914 if ((Subtarget->hasFP16())) {
2915 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
2916 }
2917 return 0;
2918}
2919
2920unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) {
2921 if ((Subtarget->hasVLX())) {
2922 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2923 }
2924 return 0;
2925}
2926
2927unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) {
2928 if ((Subtarget->hasVLX())) {
2929 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2930 }
2931 return 0;
2932}
2933
2934unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2935switch (RetVT.SimpleTy) {
2936 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2937 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2938 default: return 0;
2939}
2940}
2941
2942unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) {
2943 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2944 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2945 }
2946 return 0;
2947}
2948
2949unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) {
2950 if ((Subtarget->hasVLX())) {
2951 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2952 }
2953 return 0;
2954}
2955
2956unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) {
2957 if ((Subtarget->hasAVX512())) {
2958 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2959 }
2960 return 0;
2961}
2962
2963unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
2964switch (RetVT.SimpleTy) {
2965 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2966 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2967 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2968 default: return 0;
2969}
2970}
2971
2972unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) {
2973 if ((Subtarget->hasFP16())) {
2974 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2975 }
2976 return 0;
2977}
2978
2979unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) {
2980 if ((Subtarget->hasAVX512())) {
2981 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2982 }
2983 return 0;
2984}
2985
2986unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
2987switch (RetVT.SimpleTy) {
2988 case MVT::v16f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2989 case MVT::v16f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2990 default: return 0;
2991}
2992}
2993
2994unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
2995 if (RetVT.SimpleTy != MVT::v2f64)
2996 return 0;
2997 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2998 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2999 }
3000 return 0;
3001}
3002
3003unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) {
3004 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3005 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3006 }
3007 return 0;
3008}
3009
3010unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) {
3011 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3012 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3013 }
3014 return 0;
3015}
3016
3017unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
3018switch (RetVT.SimpleTy) {
3019 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3020 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3021 default: return 0;
3022}
3023}
3024
3025unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) {
3026 if ((Subtarget->hasFP16())) {
3027 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3028 }
3029 return 0;
3030}
3031
3032unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) {
3033 if ((Subtarget->hasDQI())) {
3034 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3035 }
3036 return 0;
3037}
3038
3039unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) {
3040 if ((Subtarget->hasDQI())) {
3041 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3042 }
3043 return 0;
3044}
3045
3046unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
3047switch (RetVT.SimpleTy) {
3048 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3049 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3050 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3051 default: return 0;
3052}
3053}
3054
3055unsigned fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
3056 switch (VT.SimpleTy) {
3057 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3058 case MVT::v16i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3059 case MVT::v32i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3060 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3061 case MVT::v8i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3062 case MVT::v16i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3063 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3064 case MVT::v4i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3065 case MVT::v8i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3066 default: return 0;
3067 }
3068}
3069
3070// FastEmit functions for ISD::TRUNCATE.
3071
3072unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0) {
3073 if (RetVT.SimpleTy != MVT::i8)
3074 return 0;
3075 if ((Subtarget->is64Bit())) {
3076 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: X86::sub_8bit);
3077 }
3078 return 0;
3079}
3080
3081unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0) {
3082 if ((Subtarget->is64Bit())) {
3083 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3084 }
3085 return 0;
3086}
3087
3088unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0) {
3089 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3090}
3091
3092unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0) {
3093switch (RetVT.SimpleTy) {
3094 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0);
3095 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0);
3096 default: return 0;
3097}
3098}
3099
3100unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0) {
3101 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3102}
3103
3104unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0) {
3105 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3106}
3107
3108unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0) {
3109 return fastEmitInst_extractsubreg(RetVT: MVT::i32, Op0, Idx: X86::sub_32bit);
3110}
3111
3112unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0) {
3113switch (RetVT.SimpleTy) {
3114 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0);
3115 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0);
3116 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0);
3117 default: return 0;
3118}
3119}
3120
3121unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
3122 if (RetVT.SimpleTy != MVT::v16i8)
3123 return 0;
3124 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3125 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ256rr, RC: &X86::VR128XRegClass, Op0);
3126 }
3127 return 0;
3128}
3129
3130unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
3131 if (RetVT.SimpleTy != MVT::v32i8)
3132 return 0;
3133 if ((Subtarget->hasBWI())) {
3134 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZrr, RC: &X86::VR256XRegClass, Op0);
3135 }
3136 return 0;
3137}
3138
3139unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
3140 if (RetVT.SimpleTy != MVT::v8i16)
3141 return 0;
3142 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3143 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ256rr, RC: &X86::VR128XRegClass, Op0);
3144 }
3145 return 0;
3146}
3147
3148unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0) {
3149 if ((Subtarget->hasAVX512())) {
3150 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZrr, RC: &X86::VR128XRegClass, Op0);
3151 }
3152 return 0;
3153}
3154
3155unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0) {
3156 if ((Subtarget->hasAVX512())) {
3157 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZrr, RC: &X86::VR256XRegClass, Op0);
3158 }
3159 return 0;
3160}
3161
3162unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
3163switch (RetVT.SimpleTy) {
3164 case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0);
3165 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0);
3166 default: return 0;
3167}
3168}
3169
3170unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
3171 if (RetVT.SimpleTy != MVT::v4i32)
3172 return 0;
3173 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3174 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ256rr, RC: &X86::VR128XRegClass, Op0);
3175 }
3176 return 0;
3177}
3178
3179unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0) {
3180 if ((Subtarget->hasAVX512())) {
3181 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZrr, RC: &X86::VR128XRegClass, Op0);
3182 }
3183 return 0;
3184}
3185
3186unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0) {
3187 if ((Subtarget->hasAVX512())) {
3188 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZrr, RC: &X86::VR256XRegClass, Op0);
3189 }
3190 return 0;
3191}
3192
3193unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
3194switch (RetVT.SimpleTy) {
3195 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0);
3196 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0);
3197 default: return 0;
3198}
3199}
3200
3201unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0) {
3202 switch (VT.SimpleTy) {
3203 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0);
3204 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0);
3205 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
3206 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0);
3207 case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0);
3208 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0);
3209 case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0);
3210 case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0);
3211 case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0);
3212 default: return 0;
3213 }
3214}
3215
3216// FastEmit functions for ISD::UINT_TO_FP.
3217
3218unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
3219 if (RetVT.SimpleTy != MVT::v8f16)
3220 return 0;
3221 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3222 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3223 }
3224 return 0;
3225}
3226
3227unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
3228 if (RetVT.SimpleTy != MVT::v16f16)
3229 return 0;
3230 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3231 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3232 }
3233 return 0;
3234}
3235
3236unsigned fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
3237 if (RetVT.SimpleTy != MVT::v32f16)
3238 return 0;
3239 if ((Subtarget->hasFP16())) {
3240 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3241 }
3242 return 0;
3243}
3244
3245unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) {
3246 if ((Subtarget->hasVLX())) {
3247 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3248 }
3249 return 0;
3250}
3251
3252unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) {
3253 if ((Subtarget->hasVLX())) {
3254 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3255 }
3256 return 0;
3257}
3258
3259unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3260switch (RetVT.SimpleTy) {
3261 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3262 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3263 default: return 0;
3264}
3265}
3266
3267unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) {
3268 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3269 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3270 }
3271 return 0;
3272}
3273
3274unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) {
3275 if ((Subtarget->hasVLX())) {
3276 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3277 }
3278 return 0;
3279}
3280
3281unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) {
3282 if ((Subtarget->hasAVX512())) {
3283 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3284 }
3285 return 0;
3286}
3287
3288unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
3289switch (RetVT.SimpleTy) {
3290 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3291 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3292 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3293 default: return 0;
3294}
3295}
3296
3297unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) {
3298 if ((Subtarget->hasFP16())) {
3299 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3300 }
3301 return 0;
3302}
3303
3304unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) {
3305 if ((Subtarget->hasAVX512())) {
3306 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3307 }
3308 return 0;
3309}
3310
3311unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
3312switch (RetVT.SimpleTy) {
3313 case MVT::v16f16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3314 case MVT::v16f32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3315 default: return 0;
3316}
3317}
3318
3319unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
3320 if (RetVT.SimpleTy != MVT::v2f64)
3321 return 0;
3322 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3323 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3324 }
3325 return 0;
3326}
3327
3328unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) {
3329 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3330 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3331 }
3332 return 0;
3333}
3334
3335unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) {
3336 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3337 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3338 }
3339 return 0;
3340}
3341
3342unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
3343switch (RetVT.SimpleTy) {
3344 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3345 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3346 default: return 0;
3347}
3348}
3349
3350unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) {
3351 if ((Subtarget->hasFP16())) {
3352 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3353 }
3354 return 0;
3355}
3356
3357unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) {
3358 if ((Subtarget->hasDQI())) {
3359 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3360 }
3361 return 0;
3362}
3363
3364unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) {
3365 if ((Subtarget->hasDQI())) {
3366 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3367 }
3368 return 0;
3369}
3370
3371unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
3372switch (RetVT.SimpleTy) {
3373 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3374 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3375 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3376 default: return 0;
3377}
3378}
3379
3380unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
3381 switch (VT.SimpleTy) {
3382 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3383 case MVT::v16i16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3384 case MVT::v32i16: return fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3385 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3386 case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3387 case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3388 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3389 case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3390 case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3391 default: return 0;
3392 }
3393}
3394
3395// FastEmit functions for ISD::ZERO_EXTEND.
3396
3397unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0) {
3398 if (RetVT.SimpleTy != MVT::i32)
3399 return 0;
3400 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
3401}
3402
3403unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0) {
3404 if (RetVT.SimpleTy != MVT::i32)
3405 return 0;
3406 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr16, RC: &X86::GR32RegClass, Op0);
3407}
3408
3409unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(unsigned Op0) {
3410 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3411 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWYrr, RC: &X86::VR256RegClass, Op0);
3412 }
3413 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3414 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
3415 }
3416 return 0;
3417}
3418
3419unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(unsigned Op0) {
3420 if ((Subtarget->hasAVX512())) {
3421 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZrr, RC: &X86::VR512RegClass, Op0);
3422 }
3423 return 0;
3424}
3425
3426unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
3427switch (RetVT.SimpleTy) {
3428 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
3429 case MVT::v16i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
3430 default: return 0;
3431}
3432}
3433
3434unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(MVT RetVT, unsigned Op0) {
3435 if (RetVT.SimpleTy != MVT::v32i16)
3436 return 0;
3437 if ((Subtarget->hasBWI())) {
3438 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZrr, RC: &X86::VR512RegClass, Op0);
3439 }
3440 return 0;
3441}
3442
3443unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(unsigned Op0) {
3444 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3445 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDYrr, RC: &X86::VR256RegClass, Op0);
3446 }
3447 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3448 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
3449 }
3450 return 0;
3451}
3452
3453unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(unsigned Op0) {
3454 if ((Subtarget->hasAVX512())) {
3455 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZrr, RC: &X86::VR512RegClass, Op0);
3456 }
3457 return 0;
3458}
3459
3460unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
3461switch (RetVT.SimpleTy) {
3462 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
3463 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
3464 default: return 0;
3465}
3466}
3467
3468unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
3469 if (RetVT.SimpleTy != MVT::v16i32)
3470 return 0;
3471 if ((Subtarget->hasAVX512())) {
3472 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZrr, RC: &X86::VR512RegClass, Op0);
3473 }
3474 return 0;
3475}
3476
3477unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3478 if (RetVT.SimpleTy != MVT::v4i64)
3479 return 0;
3480 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3481 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQYrr, RC: &X86::VR256RegClass, Op0);
3482 }
3483 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3484 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
3485 }
3486 return 0;
3487}
3488
3489unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
3490 if (RetVT.SimpleTy != MVT::v8i64)
3491 return 0;
3492 if ((Subtarget->hasAVX512())) {
3493 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZrr, RC: &X86::VR512RegClass, Op0);
3494 }
3495 return 0;
3496}
3497
3498unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
3499 switch (VT.SimpleTy) {
3500 case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0);
3501 case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0);
3502 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(RetVT, Op0);
3503 case MVT::v32i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(RetVT, Op0);
3504 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(RetVT, Op0);
3505 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(RetVT, Op0);
3506 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(RetVT, Op0);
3507 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(RetVT, Op0);
3508 default: return 0;
3509 }
3510}
3511
3512// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
3513
3514unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0) {
3515 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3516 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3517 }
3518 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3519 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3520 }
3521 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3522 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
3523 }
3524 return 0;
3525}
3526
3527unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0) {
3528 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3529 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3530 }
3531 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3532 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3533 }
3534 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3535 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
3536 }
3537 return 0;
3538}
3539
3540unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(unsigned Op0) {
3541 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3542 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDYrr, RC: &X86::VR256RegClass, Op0);
3543 }
3544 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3545 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
3546 }
3547 return 0;
3548}
3549
3550unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0) {
3551 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3552 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3553 }
3554 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3555 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3556 }
3557 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3558 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
3559 }
3560 return 0;
3561}
3562
3563unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(unsigned Op0) {
3564 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3565 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQYrr, RC: &X86::VR256RegClass, Op0);
3566 }
3567 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3568 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
3569 }
3570 return 0;
3571}
3572
3573unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(unsigned Op0) {
3574 if ((Subtarget->hasAVX512())) {
3575 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZrr, RC: &X86::VR512RegClass, Op0);
3576 }
3577 return 0;
3578}
3579
3580unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
3581switch (RetVT.SimpleTy) {
3582 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
3583 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
3584 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
3585 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
3586 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
3587 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
3588 default: return 0;
3589}
3590}
3591
3592unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0) {
3593 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3594 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3595 }
3596 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3597 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3598 }
3599 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3600 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
3601 }
3602 return 0;
3603}
3604
3605unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0) {
3606 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3607 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3608 }
3609 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3610 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3611 }
3612 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3613 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
3614 }
3615 return 0;
3616}
3617
3618unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(unsigned Op0) {
3619 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3620 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQYrr, RC: &X86::VR256RegClass, Op0);
3621 }
3622 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3623 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
3624 }
3625 return 0;
3626}
3627
3628unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
3629switch (RetVT.SimpleTy) {
3630 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
3631 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
3632 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
3633 default: return 0;
3634}
3635}
3636
3637unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3638 if (RetVT.SimpleTy != MVT::v2i64)
3639 return 0;
3640 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3641 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3642 }
3643 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3644 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3645 }
3646 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3647 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
3648 }
3649 return 0;
3650}
3651
3652unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0) {
3653 switch (VT.SimpleTy) {
3654 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
3655 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
3656 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
3657 default: return 0;
3658 }
3659}
3660
3661// FastEmit functions for X86ISD::CALL.
3662
3663unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0) {
3664 if (RetVT.SimpleTy != MVT::isVoid)
3665 return 0;
3666 if ((!Subtarget->is64Bit())) {
3667 return fastEmitInst_r(MachineInstOpcode: X86::CALL16r, RC: &X86::GR16RegClass, Op0);
3668 }
3669 return 0;
3670}
3671
3672unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0) {
3673 if (RetVT.SimpleTy != MVT::isVoid)
3674 return 0;
3675 if ((!Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3676 return fastEmitInst_r(MachineInstOpcode: X86::INDIRECT_THUNK_CALL32, RC: &X86::GR32RegClass, Op0);
3677 }
3678 if ((!Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3679 return fastEmitInst_r(MachineInstOpcode: X86::CALL32r, RC: &X86::GR32RegClass, Op0);
3680 }
3681 return 0;
3682}
3683
3684unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0) {
3685 if (RetVT.SimpleTy != MVT::isVoid)
3686 return 0;
3687 if ((Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3688 return fastEmitInst_r(MachineInstOpcode: X86::INDIRECT_THUNK_CALL64, RC: &X86::GR64RegClass, Op0);
3689 }
3690 if ((Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3691 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r, RC: &X86::GR64RegClass, Op0);
3692 }
3693 return 0;
3694}
3695
3696unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0) {
3697 switch (VT.SimpleTy) {
3698 case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0);
3699 case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0);
3700 case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0);
3701 default: return 0;
3702 }
3703}
3704
3705// FastEmit functions for X86ISD::CONFLICT.
3706
3707unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3708 if (RetVT.SimpleTy != MVT::v4i32)
3709 return 0;
3710 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3711 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ128rr, RC: &X86::VR128XRegClass, Op0);
3712 }
3713 return 0;
3714}
3715
3716unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
3717 if (RetVT.SimpleTy != MVT::v8i32)
3718 return 0;
3719 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3720 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ256rr, RC: &X86::VR256XRegClass, Op0);
3721 }
3722 return 0;
3723}
3724
3725unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
3726 if (RetVT.SimpleTy != MVT::v16i32)
3727 return 0;
3728 if ((Subtarget->hasCDI())) {
3729 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZrr, RC: &X86::VR512RegClass, Op0);
3730 }
3731 return 0;
3732}
3733
3734unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
3735 if (RetVT.SimpleTy != MVT::v2i64)
3736 return 0;
3737 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3738 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ128rr, RC: &X86::VR128XRegClass, Op0);
3739 }
3740 return 0;
3741}
3742
3743unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
3744 if (RetVT.SimpleTy != MVT::v4i64)
3745 return 0;
3746 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3747 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ256rr, RC: &X86::VR256XRegClass, Op0);
3748 }
3749 return 0;
3750}
3751
3752unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
3753 if (RetVT.SimpleTy != MVT::v8i64)
3754 return 0;
3755 if ((Subtarget->hasCDI())) {
3756 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZrr, RC: &X86::VR512RegClass, Op0);
3757 }
3758 return 0;
3759}
3760
3761unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0) {
3762 switch (VT.SimpleTy) {
3763 case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0);
3764 case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0);
3765 case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0);
3766 case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0);
3767 case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0);
3768 case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0);
3769 default: return 0;
3770 }
3771}
3772
3773// FastEmit functions for X86ISD::CVTNEPS2BF16.
3774
3775unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3776 if (RetVT.SimpleTy != MVT::v8bf16)
3777 return 0;
3778 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3779 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0);
3780 }
3781 if ((Subtarget->hasAVXNECONVERT())) {
3782 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16rr, RC: &X86::VR128RegClass, Op0);
3783 }
3784 return 0;
3785}
3786
3787unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
3788 if (RetVT.SimpleTy != MVT::v8bf16)
3789 return 0;
3790 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3791 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
3792 }
3793 return 0;
3794}
3795
3796unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
3797 if (RetVT.SimpleTy != MVT::v16bf16)
3798 return 0;
3799 if ((Subtarget->hasBF16())) {
3800 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
3801 }
3802 return 0;
3803}
3804
3805unsigned fastEmit_X86ISD_CVTNEPS2BF16_r(MVT VT, MVT RetVT, unsigned Op0) {
3806 switch (VT.SimpleTy) {
3807 case MVT::v4f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(RetVT, Op0);
3808 case MVT::v8f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(RetVT, Op0);
3809 case MVT::v16f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(RetVT, Op0);
3810 default: return 0;
3811 }
3812}
3813
3814// FastEmit functions for X86ISD::CVTP2SI.
3815
3816unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
3817 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3818 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
3819 }
3820 return 0;
3821}
3822
3823unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
3824 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3825 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
3826 }
3827 return 0;
3828}
3829
3830unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
3831 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3832 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
3833 }
3834 return 0;
3835}
3836
3837unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
3838 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3839 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
3840 }
3841 return 0;
3842}
3843
3844unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
3845 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3846 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
3847 }
3848 return 0;
3849}
3850
3851unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
3852 if ((Subtarget->hasFP16())) {
3853 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
3854 }
3855 return 0;
3856}
3857
3858unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
3859switch (RetVT.SimpleTy) {
3860 case MVT::v8i16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
3861 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
3862 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
3863 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
3864 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
3865 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
3866 default: return 0;
3867}
3868}
3869
3870unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
3871 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3872 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
3873 }
3874 return 0;
3875}
3876
3877unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
3878 if ((Subtarget->hasFP16())) {
3879 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
3880 }
3881 return 0;
3882}
3883
3884unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
3885switch (RetVT.SimpleTy) {
3886 case MVT::v16i16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
3887 case MVT::v16i32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
3888 default: return 0;
3889}
3890}
3891
3892unsigned fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
3893 if (RetVT.SimpleTy != MVT::v32i16)
3894 return 0;
3895 if ((Subtarget->hasFP16())) {
3896 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
3897 }
3898 return 0;
3899}
3900
3901unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
3902 if ((Subtarget->hasVLX())) {
3903 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
3904 }
3905 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3906 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
3907 }
3908 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3909 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
3910 }
3911 return 0;
3912}
3913
3914unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
3915 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3916 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
3917 }
3918 return 0;
3919}
3920
3921unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
3922 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3923 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
3924 }
3925 return 0;
3926}
3927
3928unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3929switch (RetVT.SimpleTy) {
3930 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
3931 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
3932 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
3933 default: return 0;
3934}
3935}
3936
3937unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
3938 if ((Subtarget->hasVLX())) {
3939 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
3940 }
3941 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3942 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
3943 }
3944 return 0;
3945}
3946
3947unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
3948 if ((Subtarget->hasDQI())) {
3949 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
3950 }
3951 return 0;
3952}
3953
3954unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
3955switch (RetVT.SimpleTy) {
3956 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
3957 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
3958 default: return 0;
3959}
3960}
3961
3962unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
3963 if (RetVT.SimpleTy != MVT::v16i32)
3964 return 0;
3965 if ((Subtarget->hasAVX512())) {
3966 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
3967 }
3968 return 0;
3969}
3970
3971unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
3972 if ((Subtarget->hasVLX())) {
3973 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
3974 }
3975 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3976 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
3977 }
3978 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3979 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
3980 }
3981 return 0;
3982}
3983
3984unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
3985 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3986 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
3987 }
3988 return 0;
3989}
3990
3991unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
3992switch (RetVT.SimpleTy) {
3993 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
3994 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
3995 default: return 0;
3996}
3997}
3998
3999unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
4000 if ((Subtarget->hasVLX())) {
4001 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
4002 }
4003 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4004 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
4005 }
4006 return 0;
4007}
4008
4009unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
4010 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4011 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4012 }
4013 return 0;
4014}
4015
4016unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
4017switch (RetVT.SimpleTy) {
4018 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
4019 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
4020 default: return 0;
4021}
4022}
4023
4024unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
4025 if ((Subtarget->hasAVX512())) {
4026 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
4027 }
4028 return 0;
4029}
4030
4031unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
4032 if ((Subtarget->hasDQI())) {
4033 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
4034 }
4035 return 0;
4036}
4037
4038unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
4039switch (RetVT.SimpleTy) {
4040 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
4041 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
4042 default: return 0;
4043}
4044}
4045
4046unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0) {
4047 switch (VT.SimpleTy) {
4048 case MVT::v8f16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(RetVT, Op0);
4049 case MVT::v16f16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(RetVT, Op0);
4050 case MVT::v32f16: return fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(RetVT, Op0);
4051 case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0);
4052 case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0);
4053 case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0);
4054 case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0);
4055 case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0);
4056 case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0);
4057 default: return 0;
4058 }
4059}
4060
4061// FastEmit functions for X86ISD::CVTP2UI.
4062
4063unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
4064 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4065 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
4066 }
4067 return 0;
4068}
4069
4070unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
4071 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4072 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4073 }
4074 return 0;
4075}
4076
4077unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
4078 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4079 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4080 }
4081 return 0;
4082}
4083
4084unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
4085 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4086 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4087 }
4088 return 0;
4089}
4090
4091unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
4092 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4093 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4094 }
4095 return 0;
4096}
4097
4098unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
4099 if ((Subtarget->hasFP16())) {
4100 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
4101 }
4102 return 0;
4103}
4104
4105unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4106switch (RetVT.SimpleTy) {
4107 case MVT::v8i16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
4108 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
4109 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
4110 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
4111 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
4112 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
4113 default: return 0;
4114}
4115}
4116
4117unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
4118 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4119 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
4120 }
4121 return 0;
4122}
4123
4124unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
4125 if ((Subtarget->hasFP16())) {
4126 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
4127 }
4128 return 0;
4129}
4130
4131unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
4132switch (RetVT.SimpleTy) {
4133 case MVT::v16i16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
4134 case MVT::v16i32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
4135 default: return 0;
4136}
4137}
4138
4139unsigned fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
4140 if (RetVT.SimpleTy != MVT::v32i16)
4141 return 0;
4142 if ((Subtarget->hasFP16())) {
4143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
4144 }
4145 return 0;
4146}
4147
4148unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
4149 if ((Subtarget->hasVLX())) {
4150 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4151 }
4152 return 0;
4153}
4154
4155unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
4156 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4157 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4158 }
4159 return 0;
4160}
4161
4162unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
4163 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4164 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4165 }
4166 return 0;
4167}
4168
4169unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4170switch (RetVT.SimpleTy) {
4171 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
4172 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
4173 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
4174 default: return 0;
4175}
4176}
4177
4178unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
4179 if ((Subtarget->hasVLX())) {
4180 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4181 }
4182 return 0;
4183}
4184
4185unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
4186 if ((Subtarget->hasDQI())) {
4187 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
4188 }
4189 return 0;
4190}
4191
4192unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
4193switch (RetVT.SimpleTy) {
4194 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
4195 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
4196 default: return 0;
4197}
4198}
4199
4200unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
4201 if (RetVT.SimpleTy != MVT::v16i32)
4202 return 0;
4203 if ((Subtarget->hasAVX512())) {
4204 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
4205 }
4206 return 0;
4207}
4208
4209unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
4210 if ((Subtarget->hasVLX())) {
4211 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4212 }
4213 return 0;
4214}
4215
4216unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
4217 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4218 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4219 }
4220 return 0;
4221}
4222
4223unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4224switch (RetVT.SimpleTy) {
4225 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
4226 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
4227 default: return 0;
4228}
4229}
4230
4231unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
4232 if ((Subtarget->hasVLX())) {
4233 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
4234 }
4235 return 0;
4236}
4237
4238unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
4239 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4240 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4241 }
4242 return 0;
4243}
4244
4245unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
4246switch (RetVT.SimpleTy) {
4247 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
4248 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
4249 default: return 0;
4250}
4251}
4252
4253unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
4254 if ((Subtarget->hasAVX512())) {
4255 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
4256 }
4257 return 0;
4258}
4259
4260unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
4261 if ((Subtarget->hasDQI())) {
4262 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
4263 }
4264 return 0;
4265}
4266
4267unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
4268switch (RetVT.SimpleTy) {
4269 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
4270 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
4271 default: return 0;
4272}
4273}
4274
4275unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0) {
4276 switch (VT.SimpleTy) {
4277 case MVT::v8f16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(RetVT, Op0);
4278 case MVT::v16f16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(RetVT, Op0);
4279 case MVT::v32f16: return fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(RetVT, Op0);
4280 case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0);
4281 case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0);
4282 case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0);
4283 case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0);
4284 case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0);
4285 case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0);
4286 default: return 0;
4287 }
4288}
4289
4290// FastEmit functions for X86ISD::CVTPH2PS.
4291
4292unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0) {
4293 if ((Subtarget->hasVLX())) {
4294 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4295 }
4296 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4297 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
4298 }
4299 return 0;
4300}
4301
4302unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0) {
4303 if ((Subtarget->hasVLX())) {
4304 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
4305 }
4306 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4307 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
4308 }
4309 return 0;
4310}
4311
4312unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
4313switch (RetVT.SimpleTy) {
4314 case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
4315 case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
4316 default: return 0;
4317}
4318}
4319
4320unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
4321 if (RetVT.SimpleTy != MVT::v16f32)
4322 return 0;
4323 if ((Subtarget->hasAVX512())) {
4324 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
4325 }
4326 return 0;
4327}
4328
4329unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0) {
4330 switch (VT.SimpleTy) {
4331 case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
4332 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
4333 default: return 0;
4334 }
4335}
4336
4337// FastEmit functions for X86ISD::CVTPH2PS_SAE.
4338
4339unsigned fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
4340 if (RetVT.SimpleTy != MVT::v16f32)
4341 return 0;
4342 if ((Subtarget->hasAVX512())) {
4343 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrrb, RC: &X86::VR512RegClass, Op0);
4344 }
4345 return 0;
4346}
4347
4348unsigned fastEmit_X86ISD_CVTPH2PS_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
4349 switch (VT.SimpleTy) {
4350 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(RetVT, Op0);
4351 default: return 0;
4352 }
4353}
4354
4355// FastEmit functions for X86ISD::CVTS2SI.
4356
4357unsigned fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(unsigned Op0) {
4358 if ((Subtarget->hasFP16())) {
4359 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4360 }
4361 return 0;
4362}
4363
4364unsigned fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(unsigned Op0) {
4365 if ((Subtarget->hasFP16())) {
4366 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4367 }
4368 return 0;
4369}
4370
4371unsigned fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4372switch (RetVT.SimpleTy) {
4373 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Op0);
4374 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Op0);
4375 default: return 0;
4376}
4377}
4378
4379unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0) {
4380 if ((Subtarget->hasAVX512())) {
4381 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4382 }
4383 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4384 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4385 }
4386 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4387 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4388 }
4389 return 0;
4390}
4391
4392unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0) {
4393 if ((Subtarget->hasAVX512())) {
4394 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4395 }
4396 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4397 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4398 }
4399 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4400 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4401 }
4402 return 0;
4403}
4404
4405unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4406switch (RetVT.SimpleTy) {
4407 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0);
4408 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0);
4409 default: return 0;
4410}
4411}
4412
4413unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0) {
4414 if ((Subtarget->hasAVX512())) {
4415 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4416 }
4417 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4418 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4419 }
4420 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4421 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4422 }
4423 return 0;
4424}
4425
4426unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0) {
4427 if ((Subtarget->hasAVX512())) {
4428 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4429 }
4430 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4431 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4432 }
4433 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4434 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4435 }
4436 return 0;
4437}
4438
4439unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4440switch (RetVT.SimpleTy) {
4441 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0);
4442 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0);
4443 default: return 0;
4444}
4445}
4446
4447unsigned fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, unsigned Op0) {
4448 switch (VT.SimpleTy) {
4449 case MVT::v8f16: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(RetVT, Op0);
4450 case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0);
4451 case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0);
4452 default: return 0;
4453 }
4454}
4455
4456// FastEmit functions for X86ISD::CVTS2UI.
4457
4458unsigned fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(unsigned Op0) {
4459 if ((Subtarget->hasFP16())) {
4460 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4461 }
4462 return 0;
4463}
4464
4465unsigned fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(unsigned Op0) {
4466 if ((Subtarget->hasFP16())) {
4467 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4468 }
4469 return 0;
4470}
4471
4472unsigned fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4473switch (RetVT.SimpleTy) {
4474 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Op0);
4475 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Op0);
4476 default: return 0;
4477}
4478}
4479
4480unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0) {
4481 if ((Subtarget->hasAVX512())) {
4482 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4483 }
4484 return 0;
4485}
4486
4487unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0) {
4488 if ((Subtarget->hasAVX512())) {
4489 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4490 }
4491 return 0;
4492}
4493
4494unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4495switch (RetVT.SimpleTy) {
4496 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0);
4497 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0);
4498 default: return 0;
4499}
4500}
4501
4502unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0) {
4503 if ((Subtarget->hasAVX512())) {
4504 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4505 }
4506 return 0;
4507}
4508
4509unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0) {
4510 if ((Subtarget->hasAVX512())) {
4511 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4512 }
4513 return 0;
4514}
4515
4516unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4517switch (RetVT.SimpleTy) {
4518 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0);
4519 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0);
4520 default: return 0;
4521}
4522}
4523
4524unsigned fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, unsigned Op0) {
4525 switch (VT.SimpleTy) {
4526 case MVT::v8f16: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(RetVT, Op0);
4527 case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0);
4528 case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0);
4529 default: return 0;
4530 }
4531}
4532
4533// FastEmit functions for X86ISD::CVTSI2P.
4534
4535unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
4536 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4537 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4538 }
4539 return 0;
4540}
4541
4542unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
4543 if ((Subtarget->hasVLX())) {
4544 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
4545 }
4546 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4547 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4548 }
4549 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4550 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4551 }
4552 return 0;
4553}
4554
4555unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
4556switch (RetVT.SimpleTy) {
4557 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
4558 case MVT::v2f64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
4559 default: return 0;
4560}
4561}
4562
4563unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
4564 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4565 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4566 }
4567 return 0;
4568}
4569
4570unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
4571 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4572 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4573 }
4574 return 0;
4575}
4576
4577unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
4578switch (RetVT.SimpleTy) {
4579 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
4580 case MVT::v4f32: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
4581 default: return 0;
4582}
4583}
4584
4585unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
4586 if (RetVT.SimpleTy != MVT::v8f16)
4587 return 0;
4588 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4589 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
4590 }
4591 return 0;
4592}
4593
4594unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0) {
4595 switch (VT.SimpleTy) {
4596 case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
4597 case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
4598 case MVT::v4i64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
4599 default: return 0;
4600 }
4601}
4602
4603// FastEmit functions for X86ISD::CVTTP2SI.
4604
4605unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
4606 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4607 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
4608 }
4609 return 0;
4610}
4611
4612unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
4613 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4614 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4615 }
4616 return 0;
4617}
4618
4619unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
4620 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4621 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4622 }
4623 return 0;
4624}
4625
4626unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
4627 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4628 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4629 }
4630 return 0;
4631}
4632
4633unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
4634 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4635 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4636 }
4637 return 0;
4638}
4639
4640unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
4641 if ((Subtarget->hasFP16())) {
4642 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
4643 }
4644 return 0;
4645}
4646
4647unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4648switch (RetVT.SimpleTy) {
4649 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
4650 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
4651 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
4652 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
4653 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
4654 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
4655 default: return 0;
4656}
4657}
4658
4659unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
4660 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4661 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
4662 }
4663 return 0;
4664}
4665
4666unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
4667 if ((Subtarget->hasFP16())) {
4668 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
4669 }
4670 return 0;
4671}
4672
4673unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
4674switch (RetVT.SimpleTy) {
4675 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
4676 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
4677 default: return 0;
4678}
4679}
4680
4681unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
4682 if (RetVT.SimpleTy != MVT::v32i16)
4683 return 0;
4684 if ((Subtarget->hasFP16())) {
4685 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
4686 }
4687 return 0;
4688}
4689
4690unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
4691 if ((Subtarget->hasVLX())) {
4692 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4693 }
4694 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4695 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4696 }
4697 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4698 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4699 }
4700 return 0;
4701}
4702
4703unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
4704 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4705 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4706 }
4707 return 0;
4708}
4709
4710unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
4711 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4712 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4713 }
4714 return 0;
4715}
4716
4717unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4718switch (RetVT.SimpleTy) {
4719 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
4720 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
4721 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
4722 default: return 0;
4723}
4724}
4725
4726unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
4727 if ((Subtarget->hasVLX())) {
4728 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4729 }
4730 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4731 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
4732 }
4733 return 0;
4734}
4735
4736unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
4737 if ((Subtarget->hasDQI())) {
4738 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
4739 }
4740 return 0;
4741}
4742
4743unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
4744switch (RetVT.SimpleTy) {
4745 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
4746 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
4747 default: return 0;
4748}
4749}
4750
4751unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
4752 if (RetVT.SimpleTy != MVT::v16i32)
4753 return 0;
4754 if ((Subtarget->hasAVX512())) {
4755 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
4756 }
4757 return 0;
4758}
4759
4760unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
4761 if ((Subtarget->hasVLX())) {
4762 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4763 }
4764 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4765 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4766 }
4767 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4768 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4769 }
4770 return 0;
4771}
4772
4773unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
4774 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4775 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4776 }
4777 return 0;
4778}
4779
4780unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4781switch (RetVT.SimpleTy) {
4782 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
4783 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
4784 default: return 0;
4785}
4786}
4787
4788unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
4789 if ((Subtarget->hasVLX())) {
4790 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
4791 }
4792 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4793 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
4794 }
4795 return 0;
4796}
4797
4798unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
4799 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4800 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4801 }
4802 return 0;
4803}
4804
4805unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
4806switch (RetVT.SimpleTy) {
4807 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
4808 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
4809 default: return 0;
4810}
4811}
4812
4813unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
4814 if ((Subtarget->hasAVX512())) {
4815 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
4816 }
4817 return 0;
4818}
4819
4820unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
4821 if ((Subtarget->hasDQI())) {
4822 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
4823 }
4824 return 0;
4825}
4826
4827unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
4828switch (RetVT.SimpleTy) {
4829 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
4830 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
4831 default: return 0;
4832}
4833}
4834
4835unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0) {
4836 switch (VT.SimpleTy) {
4837 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
4838 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
4839 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
4840 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
4841 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
4842 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
4843 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
4844 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
4845 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
4846 default: return 0;
4847 }
4848}
4849
4850// FastEmit functions for X86ISD::CVTTP2SI_SAE.
4851
4852unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4853 if (RetVT.SimpleTy != MVT::v8i64)
4854 return 0;
4855 if ((Subtarget->hasFP16())) {
4856 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrrb, RC: &X86::VR512RegClass, Op0);
4857 }
4858 return 0;
4859}
4860
4861unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
4862 if (RetVT.SimpleTy != MVT::v16i32)
4863 return 0;
4864 if ((Subtarget->hasFP16())) {
4865 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrrb, RC: &X86::VR512RegClass, Op0);
4866 }
4867 return 0;
4868}
4869
4870unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
4871 if (RetVT.SimpleTy != MVT::v32i16)
4872 return 0;
4873 if ((Subtarget->hasFP16())) {
4874 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrrb, RC: &X86::VR512RegClass, Op0);
4875 }
4876 return 0;
4877}
4878
4879unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
4880 if (RetVT.SimpleTy != MVT::v8i64)
4881 return 0;
4882 if ((Subtarget->hasDQI())) {
4883 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrrb, RC: &X86::VR512RegClass, Op0);
4884 }
4885 return 0;
4886}
4887
4888unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
4889 if (RetVT.SimpleTy != MVT::v16i32)
4890 return 0;
4891 if ((Subtarget->hasAVX512())) {
4892 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrrb, RC: &X86::VR512RegClass, Op0);
4893 }
4894 return 0;
4895}
4896
4897unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
4898 if ((Subtarget->hasAVX512())) {
4899 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrrb, RC: &X86::VR256XRegClass, Op0);
4900 }
4901 return 0;
4902}
4903
4904unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
4905 if ((Subtarget->hasDQI())) {
4906 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrrb, RC: &X86::VR512RegClass, Op0);
4907 }
4908 return 0;
4909}
4910
4911unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
4912switch (RetVT.SimpleTy) {
4913 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
4914 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
4915 default: return 0;
4916}
4917}
4918
4919unsigned fastEmit_X86ISD_CVTTP2SI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
4920 switch (VT.SimpleTy) {
4921 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(RetVT, Op0);
4922 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(RetVT, Op0);
4923 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(RetVT, Op0);
4924 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(RetVT, Op0);
4925 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(RetVT, Op0);
4926 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(RetVT, Op0);
4927 default: return 0;
4928 }
4929}
4930
4931// FastEmit functions for X86ISD::CVTTP2UI.
4932
4933unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
4934 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4935 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
4936 }
4937 return 0;
4938}
4939
4940unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
4941 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4942 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4943 }
4944 return 0;
4945}
4946
4947unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
4948 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4949 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4950 }
4951 return 0;
4952}
4953
4954unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
4955 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4956 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4957 }
4958 return 0;
4959}
4960
4961unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
4962 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4963 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4964 }
4965 return 0;
4966}
4967
4968unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
4969 if ((Subtarget->hasFP16())) {
4970 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
4971 }
4972 return 0;
4973}
4974
4975unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4976switch (RetVT.SimpleTy) {
4977 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
4978 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
4979 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
4980 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
4981 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
4982 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
4983 default: return 0;
4984}
4985}
4986
4987unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
4988 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4989 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
4990 }
4991 return 0;
4992}
4993
4994unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
4995 if ((Subtarget->hasFP16())) {
4996 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
4997 }
4998 return 0;
4999}
5000
5001unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
5002switch (RetVT.SimpleTy) {
5003 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
5004 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
5005 default: return 0;
5006}
5007}
5008
5009unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
5010 if (RetVT.SimpleTy != MVT::v32i16)
5011 return 0;
5012 if ((Subtarget->hasFP16())) {
5013 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
5014 }
5015 return 0;
5016}
5017
5018unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
5019 if ((Subtarget->hasVLX())) {
5020 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5021 }
5022 return 0;
5023}
5024
5025unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
5026 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5027 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5028 }
5029 return 0;
5030}
5031
5032unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
5033 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5034 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5035 }
5036 return 0;
5037}
5038
5039unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5040switch (RetVT.SimpleTy) {
5041 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
5042 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
5043 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
5044 default: return 0;
5045}
5046}
5047
5048unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
5049 if ((Subtarget->hasVLX())) {
5050 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5051 }
5052 return 0;
5053}
5054
5055unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
5056 if ((Subtarget->hasDQI())) {
5057 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
5058 }
5059 return 0;
5060}
5061
5062unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
5063switch (RetVT.SimpleTy) {
5064 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
5065 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
5066 default: return 0;
5067}
5068}
5069
5070unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
5071 if (RetVT.SimpleTy != MVT::v16i32)
5072 return 0;
5073 if ((Subtarget->hasAVX512())) {
5074 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
5075 }
5076 return 0;
5077}
5078
5079unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
5080 if ((Subtarget->hasVLX())) {
5081 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5082 }
5083 return 0;
5084}
5085
5086unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
5087 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5088 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5089 }
5090 return 0;
5091}
5092
5093unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5094switch (RetVT.SimpleTy) {
5095 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
5096 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
5097 default: return 0;
5098}
5099}
5100
5101unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
5102 if ((Subtarget->hasVLX())) {
5103 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
5104 }
5105 return 0;
5106}
5107
5108unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
5109 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5110 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5111 }
5112 return 0;
5113}
5114
5115unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
5116switch (RetVT.SimpleTy) {
5117 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
5118 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
5119 default: return 0;
5120}
5121}
5122
5123unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
5124 if ((Subtarget->hasAVX512())) {
5125 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
5126 }
5127 return 0;
5128}
5129
5130unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
5131 if ((Subtarget->hasDQI())) {
5132 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
5133 }
5134 return 0;
5135}
5136
5137unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5138switch (RetVT.SimpleTy) {
5139 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
5140 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
5141 default: return 0;
5142}
5143}
5144
5145unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0) {
5146 switch (VT.SimpleTy) {
5147 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
5148 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
5149 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
5150 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
5151 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
5152 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
5153 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
5154 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
5155 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
5156 default: return 0;
5157 }
5158}
5159
5160// FastEmit functions for X86ISD::CVTTP2UI_SAE.
5161
5162unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5163 if (RetVT.SimpleTy != MVT::v8i64)
5164 return 0;
5165 if ((Subtarget->hasFP16())) {
5166 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrrb, RC: &X86::VR512RegClass, Op0);
5167 }
5168 return 0;
5169}
5170
5171unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
5172 if (RetVT.SimpleTy != MVT::v16i32)
5173 return 0;
5174 if ((Subtarget->hasFP16())) {
5175 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrrb, RC: &X86::VR512RegClass, Op0);
5176 }
5177 return 0;
5178}
5179
5180unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
5181 if (RetVT.SimpleTy != MVT::v32i16)
5182 return 0;
5183 if ((Subtarget->hasFP16())) {
5184 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrrb, RC: &X86::VR512RegClass, Op0);
5185 }
5186 return 0;
5187}
5188
5189unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
5190 if (RetVT.SimpleTy != MVT::v8i64)
5191 return 0;
5192 if ((Subtarget->hasDQI())) {
5193 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrrb, RC: &X86::VR512RegClass, Op0);
5194 }
5195 return 0;
5196}
5197
5198unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
5199 if (RetVT.SimpleTy != MVT::v16i32)
5200 return 0;
5201 if ((Subtarget->hasAVX512())) {
5202 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrrb, RC: &X86::VR512RegClass, Op0);
5203 }
5204 return 0;
5205}
5206
5207unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
5208 if ((Subtarget->hasAVX512())) {
5209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrrb, RC: &X86::VR256XRegClass, Op0);
5210 }
5211 return 0;
5212}
5213
5214unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
5215 if ((Subtarget->hasDQI())) {
5216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrrb, RC: &X86::VR512RegClass, Op0);
5217 }
5218 return 0;
5219}
5220
5221unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5222switch (RetVT.SimpleTy) {
5223 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5224 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5225 default: return 0;
5226}
5227}
5228
5229unsigned fastEmit_X86ISD_CVTTP2UI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
5230 switch (VT.SimpleTy) {
5231 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(RetVT, Op0);
5232 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(RetVT, Op0);
5233 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(RetVT, Op0);
5234 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(RetVT, Op0);
5235 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(RetVT, Op0);
5236 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(RetVT, Op0);
5237 default: return 0;
5238 }
5239}
5240
5241// FastEmit functions for X86ISD::CVTTS2SI.
5242
5243unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(unsigned Op0) {
5244 if ((Subtarget->hasFP16())) {
5245 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
5246 }
5247 return 0;
5248}
5249
5250unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(unsigned Op0) {
5251 if ((Subtarget->hasFP16())) {
5252 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
5253 }
5254 return 0;
5255}
5256
5257unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5258switch (RetVT.SimpleTy) {
5259 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Op0);
5260 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Op0);
5261 default: return 0;
5262}
5263}
5264
5265unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0) {
5266 if ((Subtarget->hasAVX512())) {
5267 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
5268 }
5269 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5270 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
5271 }
5272 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
5273 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
5274 }
5275 return 0;
5276}
5277
5278unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0) {
5279 if ((Subtarget->hasAVX512())) {
5280 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
5281 }
5282 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5283 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
5284 }
5285 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
5286 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
5287 }
5288 return 0;
5289}
5290
5291unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5292switch (RetVT.SimpleTy) {
5293 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0);
5294 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0);
5295 default: return 0;
5296}
5297}
5298
5299unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0) {
5300 if ((Subtarget->hasAVX512())) {
5301 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
5302 }
5303 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5304 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
5305 }
5306 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
5307 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
5308 }
5309 return 0;
5310}
5311
5312unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0) {
5313 if ((Subtarget->hasAVX512())) {
5314 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
5315 }
5316 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5317 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
5318 }
5319 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
5320 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
5321 }
5322 return 0;
5323}
5324
5325unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5326switch (RetVT.SimpleTy) {
5327 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0);
5328 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0);
5329 default: return 0;
5330}
5331}
5332
5333unsigned fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, unsigned Op0) {
5334 switch (VT.SimpleTy) {
5335 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(RetVT, Op0);
5336 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0);
5337 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0);
5338 default: return 0;
5339 }
5340}
5341
5342// FastEmit functions for X86ISD::CVTTS2SI_SAE.
5343
5344unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(unsigned Op0) {
5345 if ((Subtarget->hasFP16())) {
5346 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
5347 }
5348 return 0;
5349}
5350
5351unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(unsigned Op0) {
5352 if ((Subtarget->hasFP16())) {
5353 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
5354 }
5355 return 0;
5356}
5357
5358unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5359switch (RetVT.SimpleTy) {
5360 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Op0);
5361 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Op0);
5362 default: return 0;
5363}
5364}
5365
5366unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(unsigned Op0) {
5367 if ((Subtarget->hasAVX512())) {
5368 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
5369 }
5370 return 0;
5371}
5372
5373unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(unsigned Op0) {
5374 if ((Subtarget->hasAVX512())) {
5375 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
5376 }
5377 return 0;
5378}
5379
5380unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5381switch (RetVT.SimpleTy) {
5382 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Op0);
5383 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Op0);
5384 default: return 0;
5385}
5386}
5387
5388unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(unsigned Op0) {
5389 if ((Subtarget->hasAVX512())) {
5390 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
5391 }
5392 return 0;
5393}
5394
5395unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(unsigned Op0) {
5396 if ((Subtarget->hasAVX512())) {
5397 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
5398 }
5399 return 0;
5400}
5401
5402unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5403switch (RetVT.SimpleTy) {
5404 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Op0);
5405 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Op0);
5406 default: return 0;
5407}
5408}
5409
5410unsigned fastEmit_X86ISD_CVTTS2SI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
5411 switch (VT.SimpleTy) {
5412 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(RetVT, Op0);
5413 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(RetVT, Op0);
5414 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(RetVT, Op0);
5415 default: return 0;
5416 }
5417}
5418
5419// FastEmit functions for X86ISD::CVTTS2UI.
5420
5421unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(unsigned Op0) {
5422 if ((Subtarget->hasFP16())) {
5423 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
5424 }
5425 return 0;
5426}
5427
5428unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(unsigned Op0) {
5429 if ((Subtarget->hasFP16())) {
5430 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
5431 }
5432 return 0;
5433}
5434
5435unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5436switch (RetVT.SimpleTy) {
5437 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Op0);
5438 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Op0);
5439 default: return 0;
5440}
5441}
5442
5443unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0) {
5444 if ((Subtarget->hasAVX512())) {
5445 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
5446 }
5447 return 0;
5448}
5449
5450unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0) {
5451 if ((Subtarget->hasAVX512())) {
5452 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
5453 }
5454 return 0;
5455}
5456
5457unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5458switch (RetVT.SimpleTy) {
5459 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0);
5460 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0);
5461 default: return 0;
5462}
5463}
5464
5465unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0) {
5466 if ((Subtarget->hasAVX512())) {
5467 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
5468 }
5469 return 0;
5470}
5471
5472unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0) {
5473 if ((Subtarget->hasAVX512())) {
5474 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
5475 }
5476 return 0;
5477}
5478
5479unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5480switch (RetVT.SimpleTy) {
5481 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0);
5482 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0);
5483 default: return 0;
5484}
5485}
5486
5487unsigned fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, unsigned Op0) {
5488 switch (VT.SimpleTy) {
5489 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(RetVT, Op0);
5490 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0);
5491 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0);
5492 default: return 0;
5493 }
5494}
5495
5496// FastEmit functions for X86ISD::CVTTS2UI_SAE.
5497
5498unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(unsigned Op0) {
5499 if ((Subtarget->hasFP16())) {
5500 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
5501 }
5502 return 0;
5503}
5504
5505unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(unsigned Op0) {
5506 if ((Subtarget->hasFP16())) {
5507 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
5508 }
5509 return 0;
5510}
5511
5512unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5513switch (RetVT.SimpleTy) {
5514 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Op0);
5515 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Op0);
5516 default: return 0;
5517}
5518}
5519
5520unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(unsigned Op0) {
5521 if ((Subtarget->hasAVX512())) {
5522 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
5523 }
5524 return 0;
5525}
5526
5527unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(unsigned Op0) {
5528 if ((Subtarget->hasAVX512())) {
5529 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
5530 }
5531 return 0;
5532}
5533
5534unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5535switch (RetVT.SimpleTy) {
5536 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Op0);
5537 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Op0);
5538 default: return 0;
5539}
5540}
5541
5542unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(unsigned Op0) {
5543 if ((Subtarget->hasAVX512())) {
5544 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
5545 }
5546 return 0;
5547}
5548
5549unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(unsigned Op0) {
5550 if ((Subtarget->hasAVX512())) {
5551 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
5552 }
5553 return 0;
5554}
5555
5556unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5557switch (RetVT.SimpleTy) {
5558 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Op0);
5559 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Op0);
5560 default: return 0;
5561}
5562}
5563
5564unsigned fastEmit_X86ISD_CVTTS2UI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
5565 switch (VT.SimpleTy) {
5566 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(RetVT, Op0);
5567 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(RetVT, Op0);
5568 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(RetVT, Op0);
5569 default: return 0;
5570 }
5571}
5572
5573// FastEmit functions for X86ISD::CVTUI2P.
5574
5575unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
5576 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
5578 }
5579 return 0;
5580}
5581
5582unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
5583 if ((Subtarget->hasVLX())) {
5584 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
5585 }
5586 return 0;
5587}
5588
5589unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
5590switch (RetVT.SimpleTy) {
5591 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
5592 case MVT::v2f64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
5593 default: return 0;
5594}
5595}
5596
5597unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
5598 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5599 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
5600 }
5601 return 0;
5602}
5603
5604unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
5605 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5606 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
5607 }
5608 return 0;
5609}
5610
5611unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
5612switch (RetVT.SimpleTy) {
5613 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
5614 case MVT::v4f32: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
5615 default: return 0;
5616}
5617}
5618
5619unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
5620 if (RetVT.SimpleTy != MVT::v8f16)
5621 return 0;
5622 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5623 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
5624 }
5625 return 0;
5626}
5627
5628unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0) {
5629 switch (VT.SimpleTy) {
5630 case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
5631 case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
5632 case MVT::v4i64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
5633 default: return 0;
5634 }
5635}
5636
5637// FastEmit functions for X86ISD::DYN_ALLOCA.
5638
5639unsigned fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0) {
5640 if (RetVT.SimpleTy != MVT::isVoid)
5641 return 0;
5642 if ((!Subtarget->isTarget64BitLP64())) {
5643 return fastEmitInst_r(MachineInstOpcode: X86::DYN_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
5644 }
5645 return 0;
5646}
5647
5648unsigned fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0) {
5649 if (RetVT.SimpleTy != MVT::isVoid)
5650 return 0;
5651 if ((Subtarget->is64Bit())) {
5652 return fastEmitInst_r(MachineInstOpcode: X86::DYN_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
5653 }
5654 return 0;
5655}
5656
5657unsigned fastEmit_X86ISD_DYN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0) {
5658 switch (VT.SimpleTy) {
5659 case MVT::i32: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(RetVT, Op0);
5660 case MVT::i64: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(RetVT, Op0);
5661 default: return 0;
5662 }
5663}
5664
5665// FastEmit functions for X86ISD::EH_RETURN.
5666
5667unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0) {
5668 if (RetVT.SimpleTy != MVT::isVoid)
5669 return 0;
5670 return fastEmitInst_r(MachineInstOpcode: X86::EH_RETURN, RC: &X86::GR32RegClass, Op0);
5671}
5672
5673unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0) {
5674 if (RetVT.SimpleTy != MVT::isVoid)
5675 return 0;
5676 return fastEmitInst_r(MachineInstOpcode: X86::EH_RETURN64, RC: &X86::GR64RegClass, Op0);
5677}
5678
5679unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0) {
5680 switch (VT.SimpleTy) {
5681 case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0);
5682 case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0);
5683 default: return 0;
5684 }
5685}
5686
5687// FastEmit functions for X86ISD::FGETEXP.
5688
5689unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5690 if (RetVT.SimpleTy != MVT::v8f16)
5691 return 0;
5692 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5693 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ128r, RC: &X86::VR128XRegClass, Op0);
5694 }
5695 return 0;
5696}
5697
5698unsigned fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
5699 if (RetVT.SimpleTy != MVT::v16f16)
5700 return 0;
5701 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5702 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ256r, RC: &X86::VR256XRegClass, Op0);
5703 }
5704 return 0;
5705}
5706
5707unsigned fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
5708 if (RetVT.SimpleTy != MVT::v32f16)
5709 return 0;
5710 if ((Subtarget->hasFP16())) {
5711 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZr, RC: &X86::VR512RegClass, Op0);
5712 }
5713 return 0;
5714}
5715
5716unsigned fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5717 if (RetVT.SimpleTy != MVT::v4f32)
5718 return 0;
5719 if ((Subtarget->hasVLX())) {
5720 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ128r, RC: &X86::VR128XRegClass, Op0);
5721 }
5722 return 0;
5723}
5724
5725unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
5726 if (RetVT.SimpleTy != MVT::v8f32)
5727 return 0;
5728 if ((Subtarget->hasVLX())) {
5729 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ256r, RC: &X86::VR256XRegClass, Op0);
5730 }
5731 return 0;
5732}
5733
5734unsigned fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
5735 if (RetVT.SimpleTy != MVT::v16f32)
5736 return 0;
5737 if ((Subtarget->hasAVX512())) {
5738 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZr, RC: &X86::VR512RegClass, Op0);
5739 }
5740 return 0;
5741}
5742
5743unsigned fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5744 if (RetVT.SimpleTy != MVT::v2f64)
5745 return 0;
5746 if ((Subtarget->hasVLX())) {
5747 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ128r, RC: &X86::VR128XRegClass, Op0);
5748 }
5749 return 0;
5750}
5751
5752unsigned fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
5753 if (RetVT.SimpleTy != MVT::v4f64)
5754 return 0;
5755 if ((Subtarget->hasVLX())) {
5756 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ256r, RC: &X86::VR256XRegClass, Op0);
5757 }
5758 return 0;
5759}
5760
5761unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5762 if (RetVT.SimpleTy != MVT::v8f64)
5763 return 0;
5764 if ((Subtarget->hasAVX512())) {
5765 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZr, RC: &X86::VR512RegClass, Op0);
5766 }
5767 return 0;
5768}
5769
5770unsigned fastEmit_X86ISD_FGETEXP_r(MVT VT, MVT RetVT, unsigned Op0) {
5771 switch (VT.SimpleTy) {
5772 case MVT::v8f16: return fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(RetVT, Op0);
5773 case MVT::v16f16: return fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(RetVT, Op0);
5774 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(RetVT, Op0);
5775 case MVT::v4f32: return fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(RetVT, Op0);
5776 case MVT::v8f32: return fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(RetVT, Op0);
5777 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(RetVT, Op0);
5778 case MVT::v2f64: return fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(RetVT, Op0);
5779 case MVT::v4f64: return fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(RetVT, Op0);
5780 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(RetVT, Op0);
5781 default: return 0;
5782 }
5783}
5784
5785// FastEmit functions for X86ISD::FGETEXP_SAE.
5786
5787unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
5788 if (RetVT.SimpleTy != MVT::v32f16)
5789 return 0;
5790 if ((Subtarget->hasFP16())) {
5791 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZrb, RC: &X86::VR512RegClass, Op0);
5792 }
5793 return 0;
5794}
5795
5796unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
5797 if (RetVT.SimpleTy != MVT::v16f32)
5798 return 0;
5799 if ((Subtarget->hasAVX512())) {
5800 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZrb, RC: &X86::VR512RegClass, Op0);
5801 }
5802 return 0;
5803}
5804
5805unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5806 if (RetVT.SimpleTy != MVT::v8f64)
5807 return 0;
5808 if ((Subtarget->hasAVX512())) {
5809 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZrb, RC: &X86::VR512RegClass, Op0);
5810 }
5811 return 0;
5812}
5813
5814unsigned fastEmit_X86ISD_FGETEXP_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
5815 switch (VT.SimpleTy) {
5816 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(RetVT, Op0);
5817 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(RetVT, Op0);
5818 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(RetVT, Op0);
5819 default: return 0;
5820 }
5821}
5822
5823// FastEmit functions for X86ISD::FRCP.
5824
5825unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0) {
5826 if (RetVT.SimpleTy != MVT::f32)
5827 return 0;
5828 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5829 return fastEmitInst_r(MachineInstOpcode: X86::RCPSSr, RC: &X86::FR32RegClass, Op0);
5830 }
5831 return 0;
5832}
5833
5834unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5835 if (RetVT.SimpleTy != MVT::v4f32)
5836 return 0;
5837 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5838 return fastEmitInst_r(MachineInstOpcode: X86::RCPPSr, RC: &X86::VR128RegClass, Op0);
5839 }
5840 if ((Subtarget->hasAVX())) {
5841 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSr, RC: &X86::VR128RegClass, Op0);
5842 }
5843 return 0;
5844}
5845
5846unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
5847 if (RetVT.SimpleTy != MVT::v8f32)
5848 return 0;
5849 if ((Subtarget->hasAVX())) {
5850 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSYr, RC: &X86::VR256RegClass, Op0);
5851 }
5852 return 0;
5853}
5854
5855unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0) {
5856 switch (VT.SimpleTy) {
5857 case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0);
5858 case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0);
5859 case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0);
5860 default: return 0;
5861 }
5862}
5863
5864// FastEmit functions for X86ISD::FRSQRT.
5865
5866unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) {
5867 if (RetVT.SimpleTy != MVT::f32)
5868 return 0;
5869 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5870 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTSSr, RC: &X86::FR32RegClass, Op0);
5871 }
5872 return 0;
5873}
5874
5875unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5876 if (RetVT.SimpleTy != MVT::v4f32)
5877 return 0;
5878 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5879 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTPSr, RC: &X86::VR128RegClass, Op0);
5880 }
5881 if ((Subtarget->hasAVX())) {
5882 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSr, RC: &X86::VR128RegClass, Op0);
5883 }
5884 return 0;
5885}
5886
5887unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
5888 if (RetVT.SimpleTy != MVT::v8f32)
5889 return 0;
5890 if ((Subtarget->hasAVX())) {
5891 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSYr, RC: &X86::VR256RegClass, Op0);
5892 }
5893 return 0;
5894}
5895
5896unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0) {
5897 switch (VT.SimpleTy) {
5898 case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0);
5899 case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0);
5900 case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0);
5901 default: return 0;
5902 }
5903}
5904
5905// FastEmit functions for X86ISD::MMX_MOVD2W.
5906
5907unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0) {
5908 if (RetVT.SimpleTy != MVT::i32)
5909 return 0;
5910 if ((Subtarget->hasMMX())) {
5911 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64grr, RC: &X86::GR32RegClass, Op0);
5912 }
5913 return 0;
5914}
5915
5916unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0) {
5917 switch (VT.SimpleTy) {
5918 case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0);
5919 default: return 0;
5920 }
5921}
5922
5923// FastEmit functions for X86ISD::MMX_MOVW2D.
5924
5925unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0) {
5926 if (RetVT.SimpleTy != MVT::x86mmx)
5927 return 0;
5928 if ((Subtarget->hasMMX())) {
5929 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64rr, RC: &X86::VR64RegClass, Op0);
5930 }
5931 return 0;
5932}
5933
5934unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0) {
5935 switch (VT.SimpleTy) {
5936 case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0);
5937 default: return 0;
5938 }
5939}
5940
5941// FastEmit functions for X86ISD::MOVDDUP.
5942
5943unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5944 if (RetVT.SimpleTy != MVT::v2f64)
5945 return 0;
5946 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
5947 return fastEmitInst_r(MachineInstOpcode: X86::MOVDDUPrr, RC: &X86::VR128RegClass, Op0);
5948 }
5949 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5950 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
5951 }
5952 return 0;
5953}
5954
5955unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
5956 if (RetVT.SimpleTy != MVT::v4f64)
5957 return 0;
5958 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5959 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
5960 }
5961 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5962 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPYrr, RC: &X86::VR256RegClass, Op0);
5963 }
5964 return 0;
5965}
5966
5967unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5968 if (RetVT.SimpleTy != MVT::v8f64)
5969 return 0;
5970 if ((Subtarget->hasAVX512())) {
5971 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZrr, RC: &X86::VR512RegClass, Op0);
5972 }
5973 return 0;
5974}
5975
5976unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0) {
5977 switch (VT.SimpleTy) {
5978 case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0);
5979 case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0);
5980 case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0);
5981 default: return 0;
5982 }
5983}
5984
5985// FastEmit functions for X86ISD::MOVDQ2Q.
5986
5987unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
5988 if (RetVT.SimpleTy != MVT::x86mmx)
5989 return 0;
5990 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
5991 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVDQ2Qrr, RC: &X86::VR64RegClass, Op0);
5992 }
5993 return 0;
5994}
5995
5996unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0) {
5997 switch (VT.SimpleTy) {
5998 case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0);
5999 default: return 0;
6000 }
6001}
6002
6003// FastEmit functions for X86ISD::MOVMSK.
6004
6005unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
6006 if (RetVT.SimpleTy != MVT::i32)
6007 return 0;
6008 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6009 return fastEmitInst_r(MachineInstOpcode: X86::PMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
6010 }
6011 if ((Subtarget->hasAVX())) {
6012 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
6013 }
6014 return 0;
6015}
6016
6017unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0) {
6018 if (RetVT.SimpleTy != MVT::i32)
6019 return 0;
6020 if ((Subtarget->hasAVX2())) {
6021 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBYrr, RC: &X86::GR32RegClass, Op0);
6022 }
6023 return 0;
6024}
6025
6026unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6027 if (RetVT.SimpleTy != MVT::i32)
6028 return 0;
6029 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6030 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
6031 }
6032 if ((Subtarget->hasAVX())) {
6033 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
6034 }
6035 return 0;
6036}
6037
6038unsigned fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
6039 if (RetVT.SimpleTy != MVT::i32)
6040 return 0;
6041 if ((Subtarget->hasAVX())) {
6042 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
6043 }
6044 return 0;
6045}
6046
6047unsigned fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
6048 if (RetVT.SimpleTy != MVT::i32)
6049 return 0;
6050 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6051 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
6052 }
6053 if ((Subtarget->hasAVX())) {
6054 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
6055 }
6056 return 0;
6057}
6058
6059unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
6060 if (RetVT.SimpleTy != MVT::i32)
6061 return 0;
6062 if ((Subtarget->hasAVX())) {
6063 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
6064 }
6065 return 0;
6066}
6067
6068unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6069 if (RetVT.SimpleTy != MVT::i32)
6070 return 0;
6071 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6072 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
6073 }
6074 if ((Subtarget->hasAVX())) {
6075 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
6076 }
6077 return 0;
6078}
6079
6080unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6081 if (RetVT.SimpleTy != MVT::i32)
6082 return 0;
6083 if ((Subtarget->hasAVX())) {
6084 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
6085 }
6086 return 0;
6087}
6088
6089unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6090 if (RetVT.SimpleTy != MVT::i32)
6091 return 0;
6092 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6093 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
6094 }
6095 if ((Subtarget->hasAVX())) {
6096 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
6097 }
6098 return 0;
6099}
6100
6101unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
6102 if (RetVT.SimpleTy != MVT::i32)
6103 return 0;
6104 if ((Subtarget->hasAVX())) {
6105 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
6106 }
6107 return 0;
6108}
6109
6110unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0) {
6111 switch (VT.SimpleTy) {
6112 case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0);
6113 case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0);
6114 case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0);
6115 case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0);
6116 case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0);
6117 case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0);
6118 case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0);
6119 case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0);
6120 case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0);
6121 case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0);
6122 default: return 0;
6123 }
6124}
6125
6126// FastEmit functions for X86ISD::MOVQ2DQ.
6127
6128unsigned fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(MVT RetVT, unsigned Op0) {
6129 if (RetVT.SimpleTy != MVT::v2i64)
6130 return 0;
6131 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
6132 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2DQrr, RC: &X86::VR128RegClass, Op0);
6133 }
6134 return 0;
6135}
6136
6137unsigned fastEmit_X86ISD_MOVQ2DQ_r(MVT VT, MVT RetVT, unsigned Op0) {
6138 switch (VT.SimpleTy) {
6139 case MVT::x86mmx: return fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(RetVT, Op0);
6140 default: return 0;
6141 }
6142}
6143
6144// FastEmit functions for X86ISD::MOVSHDUP.
6145
6146unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6147 if (RetVT.SimpleTy != MVT::v4i32)
6148 return 0;
6149 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
6150 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
6151 }
6152 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6153 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
6154 }
6155 return 0;
6156}
6157
6158unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
6159 if (RetVT.SimpleTy != MVT::v8i32)
6160 return 0;
6161 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6162 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
6163 }
6164 return 0;
6165}
6166
6167unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6168 if (RetVT.SimpleTy != MVT::v4f32)
6169 return 0;
6170 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6171 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
6172 }
6173 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
6174 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
6175 }
6176 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6177 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
6178 }
6179 return 0;
6180}
6181
6182unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6183 if (RetVT.SimpleTy != MVT::v8f32)
6184 return 0;
6185 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6186 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
6187 }
6188 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6189 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
6190 }
6191 return 0;
6192}
6193
6194unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6195 if (RetVT.SimpleTy != MVT::v16f32)
6196 return 0;
6197 if ((Subtarget->hasAVX512())) {
6198 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZrr, RC: &X86::VR512RegClass, Op0);
6199 }
6200 return 0;
6201}
6202
6203unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0) {
6204 switch (VT.SimpleTy) {
6205 case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0);
6206 case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0);
6207 case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0);
6208 case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0);
6209 case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0);
6210 default: return 0;
6211 }
6212}
6213
6214// FastEmit functions for X86ISD::MOVSLDUP.
6215
6216unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6217 if (RetVT.SimpleTy != MVT::v4i32)
6218 return 0;
6219 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
6220 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
6221 }
6222 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6223 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
6224 }
6225 return 0;
6226}
6227
6228unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
6229 if (RetVT.SimpleTy != MVT::v8i32)
6230 return 0;
6231 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6232 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
6233 }
6234 return 0;
6235}
6236
6237unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6238 if (RetVT.SimpleTy != MVT::v4f32)
6239 return 0;
6240 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6241 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
6242 }
6243 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
6244 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
6245 }
6246 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6247 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
6248 }
6249 return 0;
6250}
6251
6252unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6253 if (RetVT.SimpleTy != MVT::v8f32)
6254 return 0;
6255 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6256 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
6257 }
6258 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6259 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
6260 }
6261 return 0;
6262}
6263
6264unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6265 if (RetVT.SimpleTy != MVT::v16f32)
6266 return 0;
6267 if ((Subtarget->hasAVX512())) {
6268 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZrr, RC: &X86::VR512RegClass, Op0);
6269 }
6270 return 0;
6271}
6272
6273unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0) {
6274 switch (VT.SimpleTy) {
6275 case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0);
6276 case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0);
6277 case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0);
6278 case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0);
6279 case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0);
6280 default: return 0;
6281 }
6282}
6283
6284// FastEmit functions for X86ISD::NT_BRIND.
6285
6286unsigned fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0) {
6287 if (RetVT.SimpleTy != MVT::isVoid)
6288 return 0;
6289 if ((!Subtarget->is64Bit())) {
6290 return fastEmitInst_r(MachineInstOpcode: X86::JMP16r_NT, RC: &X86::GR16RegClass, Op0);
6291 }
6292 return 0;
6293}
6294
6295unsigned fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0) {
6296 if (RetVT.SimpleTy != MVT::isVoid)
6297 return 0;
6298 if ((!Subtarget->is64Bit())) {
6299 return fastEmitInst_r(MachineInstOpcode: X86::JMP32r_NT, RC: &X86::GR32RegClass, Op0);
6300 }
6301 return 0;
6302}
6303
6304unsigned fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0) {
6305 if (RetVT.SimpleTy != MVT::isVoid)
6306 return 0;
6307 if ((Subtarget->is64Bit())) {
6308 return fastEmitInst_r(MachineInstOpcode: X86::JMP64r_NT, RC: &X86::GR64RegClass, Op0);
6309 }
6310 return 0;
6311}
6312
6313unsigned fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, unsigned Op0) {
6314 switch (VT.SimpleTy) {
6315 case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0);
6316 case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0);
6317 case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0);
6318 default: return 0;
6319 }
6320}
6321
6322// FastEmit functions for X86ISD::NT_CALL.
6323
6324unsigned fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, unsigned Op0) {
6325 if (RetVT.SimpleTy != MVT::isVoid)
6326 return 0;
6327 if ((!Subtarget->is64Bit())) {
6328 return fastEmitInst_r(MachineInstOpcode: X86::CALL16r_NT, RC: &X86::GR16RegClass, Op0);
6329 }
6330 return 0;
6331}
6332
6333unsigned fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, unsigned Op0) {
6334 if (RetVT.SimpleTy != MVT::isVoid)
6335 return 0;
6336 if ((!Subtarget->is64Bit())) {
6337 return fastEmitInst_r(MachineInstOpcode: X86::CALL32r_NT, RC: &X86::GR32RegClass, Op0);
6338 }
6339 return 0;
6340}
6341
6342unsigned fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, unsigned Op0) {
6343 if (RetVT.SimpleTy != MVT::isVoid)
6344 return 0;
6345 if ((Subtarget->is64Bit())) {
6346 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r_NT, RC: &X86::GR64RegClass, Op0);
6347 }
6348 return 0;
6349}
6350
6351unsigned fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, unsigned Op0) {
6352 switch (VT.SimpleTy) {
6353 case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0);
6354 case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0);
6355 case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0);
6356 default: return 0;
6357 }
6358}
6359
6360// FastEmit functions for X86ISD::PHMINPOS.
6361
6362unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6363 if (RetVT.SimpleTy != MVT::v8i16)
6364 return 0;
6365 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
6366 return fastEmitInst_r(MachineInstOpcode: X86::PHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
6367 }
6368 if ((Subtarget->hasAVX())) {
6369 return fastEmitInst_r(MachineInstOpcode: X86::VPHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
6370 }
6371 return 0;
6372}
6373
6374unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0) {
6375 switch (VT.SimpleTy) {
6376 case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0);
6377 default: return 0;
6378 }
6379}
6380
6381// FastEmit functions for X86ISD::PROBED_ALLOCA.
6382
6383unsigned fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0) {
6384 if (RetVT.SimpleTy != MVT::i32)
6385 return 0;
6386 if ((!Subtarget->isTarget64BitLP64())) {
6387 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
6388 }
6389 return 0;
6390}
6391
6392unsigned fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0) {
6393 if (RetVT.SimpleTy != MVT::i64)
6394 return 0;
6395 if ((Subtarget->is64Bit())) {
6396 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
6397 }
6398 return 0;
6399}
6400
6401unsigned fastEmit_X86ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0) {
6402 switch (VT.SimpleTy) {
6403 case MVT::i32: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(RetVT, Op0);
6404 case MVT::i64: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
6405 default: return 0;
6406 }
6407}
6408
6409// FastEmit functions for X86ISD::RCP14.
6410
6411unsigned fastEmit_X86ISD_RCP14_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
6412 if (RetVT.SimpleTy != MVT::v8f16)
6413 return 0;
6414 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6415 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ128r, RC: &X86::VR128XRegClass, Op0);
6416 }
6417 return 0;
6418}
6419
6420unsigned fastEmit_X86ISD_RCP14_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
6421 if (RetVT.SimpleTy != MVT::v16f16)
6422 return 0;
6423 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6424 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ256r, RC: &X86::VR256XRegClass, Op0);
6425 }
6426 return 0;
6427}
6428
6429unsigned fastEmit_X86ISD_RCP14_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
6430 if (RetVT.SimpleTy != MVT::v32f16)
6431 return 0;
6432 if ((Subtarget->hasFP16())) {
6433 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZr, RC: &X86::VR512RegClass, Op0);
6434 }
6435 return 0;
6436}
6437
6438unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6439 if (RetVT.SimpleTy != MVT::v4f32)
6440 return 0;
6441 if ((Subtarget->hasVLX())) {
6442 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ128r, RC: &X86::VR128XRegClass, Op0);
6443 }
6444 return 0;
6445}
6446
6447unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6448 if (RetVT.SimpleTy != MVT::v8f32)
6449 return 0;
6450 if ((Subtarget->hasVLX())) {
6451 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ256r, RC: &X86::VR256XRegClass, Op0);
6452 }
6453 return 0;
6454}
6455
6456unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6457 if (RetVT.SimpleTy != MVT::v16f32)
6458 return 0;
6459 if ((Subtarget->hasAVX512())) {
6460 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZr, RC: &X86::VR512RegClass, Op0);
6461 }
6462 return 0;
6463}
6464
6465unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6466 if (RetVT.SimpleTy != MVT::v2f64)
6467 return 0;
6468 if ((Subtarget->hasVLX())) {
6469 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ128r, RC: &X86::VR128XRegClass, Op0);
6470 }
6471 return 0;
6472}
6473
6474unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
6475 if (RetVT.SimpleTy != MVT::v4f64)
6476 return 0;
6477 if ((Subtarget->hasVLX())) {
6478 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ256r, RC: &X86::VR256XRegClass, Op0);
6479 }
6480 return 0;
6481}
6482
6483unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
6484 if (RetVT.SimpleTy != MVT::v8f64)
6485 return 0;
6486 if ((Subtarget->hasAVX512())) {
6487 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZr, RC: &X86::VR512RegClass, Op0);
6488 }
6489 return 0;
6490}
6491
6492unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0) {
6493 switch (VT.SimpleTy) {
6494 case MVT::v8f16: return fastEmit_X86ISD_RCP14_MVT_v8f16_r(RetVT, Op0);
6495 case MVT::v16f16: return fastEmit_X86ISD_RCP14_MVT_v16f16_r(RetVT, Op0);
6496 case MVT::v32f16: return fastEmit_X86ISD_RCP14_MVT_v32f16_r(RetVT, Op0);
6497 case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0);
6498 case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0);
6499 case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0);
6500 case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0);
6501 case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0);
6502 case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0);
6503 default: return 0;
6504 }
6505}
6506
6507// FastEmit functions for X86ISD::RSQRT14.
6508
6509unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
6510 if (RetVT.SimpleTy != MVT::v8f16)
6511 return 0;
6512 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6513 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
6514 }
6515 return 0;
6516}
6517
6518unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
6519 if (RetVT.SimpleTy != MVT::v16f16)
6520 return 0;
6521 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6522 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
6523 }
6524 return 0;
6525}
6526
6527unsigned fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
6528 if (RetVT.SimpleTy != MVT::v32f16)
6529 return 0;
6530 if ((Subtarget->hasFP16())) {
6531 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZr, RC: &X86::VR512RegClass, Op0);
6532 }
6533 return 0;
6534}
6535
6536unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6537 if (RetVT.SimpleTy != MVT::v4f32)
6538 return 0;
6539 if ((Subtarget->hasVLX())) {
6540 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ128r, RC: &X86::VR128XRegClass, Op0);
6541 }
6542 return 0;
6543}
6544
6545unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6546 if (RetVT.SimpleTy != MVT::v8f32)
6547 return 0;
6548 if ((Subtarget->hasVLX())) {
6549 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ256r, RC: &X86::VR256XRegClass, Op0);
6550 }
6551 return 0;
6552}
6553
6554unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6555 if (RetVT.SimpleTy != MVT::v16f32)
6556 return 0;
6557 if ((Subtarget->hasAVX512())) {
6558 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZr, RC: &X86::VR512RegClass, Op0);
6559 }
6560 return 0;
6561}
6562
6563unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6564 if (RetVT.SimpleTy != MVT::v2f64)
6565 return 0;
6566 if ((Subtarget->hasVLX())) {
6567 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ128r, RC: &X86::VR128XRegClass, Op0);
6568 }
6569 return 0;
6570}
6571
6572unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
6573 if (RetVT.SimpleTy != MVT::v4f64)
6574 return 0;
6575 if ((Subtarget->hasVLX())) {
6576 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ256r, RC: &X86::VR256XRegClass, Op0);
6577 }
6578 return 0;
6579}
6580
6581unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
6582 if (RetVT.SimpleTy != MVT::v8f64)
6583 return 0;
6584 if ((Subtarget->hasAVX512())) {
6585 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZr, RC: &X86::VR512RegClass, Op0);
6586 }
6587 return 0;
6588}
6589
6590unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0) {
6591 switch (VT.SimpleTy) {
6592 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(RetVT, Op0);
6593 case MVT::v16f16: return fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(RetVT, Op0);
6594 case MVT::v32f16: return fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(RetVT, Op0);
6595 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0);
6596 case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0);
6597 case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0);
6598 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0);
6599 case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0);
6600 case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0);
6601 default: return 0;
6602 }
6603}
6604
6605// FastEmit functions for X86ISD::SEG_ALLOCA.
6606
6607unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0) {
6608 if (RetVT.SimpleTy != MVT::i32)
6609 return 0;
6610 if ((!Subtarget->isTarget64BitLP64())) {
6611 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
6612 }
6613 return 0;
6614}
6615
6616unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0) {
6617 if (RetVT.SimpleTy != MVT::i64)
6618 return 0;
6619 if ((Subtarget->is64Bit())) {
6620 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
6621 }
6622 return 0;
6623}
6624
6625unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0) {
6626 switch (VT.SimpleTy) {
6627 case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0);
6628 case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0);
6629 default: return 0;
6630 }
6631}
6632
6633// FastEmit functions for X86ISD::STRICT_CVTPH2PS.
6634
6635unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0) {
6636 if ((Subtarget->hasVLX())) {
6637 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
6638 }
6639 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
6640 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
6641 }
6642 return 0;
6643}
6644
6645unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0) {
6646 if ((Subtarget->hasVLX())) {
6647 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
6648 }
6649 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
6650 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
6651 }
6652 return 0;
6653}
6654
6655unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6656switch (RetVT.SimpleTy) {
6657 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
6658 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
6659 default: return 0;
6660}
6661}
6662
6663unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
6664 if (RetVT.SimpleTy != MVT::v16f32)
6665 return 0;
6666 if ((Subtarget->hasAVX512())) {
6667 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
6668 }
6669 return 0;
6670}
6671
6672unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0) {
6673 switch (VT.SimpleTy) {
6674 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
6675 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
6676 default: return 0;
6677 }
6678}
6679
6680// FastEmit functions for X86ISD::STRICT_CVTSI2P.
6681
6682unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
6683 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6684 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6685 }
6686 return 0;
6687}
6688
6689unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
6690 if ((Subtarget->hasVLX())) {
6691 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
6692 }
6693 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6694 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
6695 }
6696 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6697 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
6698 }
6699 return 0;
6700}
6701
6702unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6703switch (RetVT.SimpleTy) {
6704 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
6705 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
6706 default: return 0;
6707}
6708}
6709
6710unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
6711 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6712 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6713 }
6714 return 0;
6715}
6716
6717unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
6718 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6719 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
6720 }
6721 return 0;
6722}
6723
6724unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
6725switch (RetVT.SimpleTy) {
6726 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
6727 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
6728 default: return 0;
6729}
6730}
6731
6732unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
6733 if (RetVT.SimpleTy != MVT::v8f16)
6734 return 0;
6735 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6736 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
6737 }
6738 return 0;
6739}
6740
6741unsigned fastEmit_X86ISD_STRICT_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0) {
6742 switch (VT.SimpleTy) {
6743 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
6744 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
6745 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
6746 default: return 0;
6747 }
6748}
6749
6750// FastEmit functions for X86ISD::STRICT_CVTTP2SI.
6751
6752unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
6753 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6754 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
6755 }
6756 return 0;
6757}
6758
6759unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
6760 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6761 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
6762 }
6763 return 0;
6764}
6765
6766unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
6767 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6768 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
6769 }
6770 return 0;
6771}
6772
6773unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
6774 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6775 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
6776 }
6777 return 0;
6778}
6779
6780unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
6781 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6782 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
6783 }
6784 return 0;
6785}
6786
6787unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
6788 if ((Subtarget->hasFP16())) {
6789 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
6790 }
6791 return 0;
6792}
6793
6794unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
6795switch (RetVT.SimpleTy) {
6796 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
6797 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
6798 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
6799 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
6800 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
6801 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
6802 default: return 0;
6803}
6804}
6805
6806unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
6807 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6808 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
6809 }
6810 return 0;
6811}
6812
6813unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
6814 if ((Subtarget->hasFP16())) {
6815 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
6816 }
6817 return 0;
6818}
6819
6820unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
6821switch (RetVT.SimpleTy) {
6822 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
6823 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
6824 default: return 0;
6825}
6826}
6827
6828unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
6829 if (RetVT.SimpleTy != MVT::v32i16)
6830 return 0;
6831 if ((Subtarget->hasFP16())) {
6832 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
6833 }
6834 return 0;
6835}
6836
6837unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
6838 if ((Subtarget->hasVLX())) {
6839 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
6840 }
6841 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6842 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
6843 }
6844 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6845 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
6846 }
6847 return 0;
6848}
6849
6850unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
6851 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6852 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
6853 }
6854 return 0;
6855}
6856
6857unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
6858 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6859 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
6860 }
6861 return 0;
6862}
6863
6864unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6865switch (RetVT.SimpleTy) {
6866 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
6867 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
6868 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
6869 default: return 0;
6870}
6871}
6872
6873unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
6874 if ((Subtarget->hasVLX())) {
6875 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
6876 }
6877 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6878 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
6879 }
6880 return 0;
6881}
6882
6883unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
6884 if ((Subtarget->hasDQI())) {
6885 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
6886 }
6887 return 0;
6888}
6889
6890unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6891switch (RetVT.SimpleTy) {
6892 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
6893 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
6894 default: return 0;
6895}
6896}
6897
6898unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6899 if (RetVT.SimpleTy != MVT::v16i32)
6900 return 0;
6901 if ((Subtarget->hasAVX512())) {
6902 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
6903 }
6904 return 0;
6905}
6906
6907unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
6908 if ((Subtarget->hasVLX())) {
6909 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
6910 }
6911 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6912 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
6913 }
6914 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6915 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
6916 }
6917 return 0;
6918}
6919
6920unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
6921 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6922 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
6923 }
6924 return 0;
6925}
6926
6927unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6928switch (RetVT.SimpleTy) {
6929 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
6930 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
6931 default: return 0;
6932}
6933}
6934
6935unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
6936 if ((Subtarget->hasVLX())) {
6937 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
6938 }
6939 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6940 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
6941 }
6942 return 0;
6943}
6944
6945unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
6946 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6947 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
6948 }
6949 return 0;
6950}
6951
6952unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
6953switch (RetVT.SimpleTy) {
6954 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
6955 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
6956 default: return 0;
6957}
6958}
6959
6960unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
6961 if ((Subtarget->hasAVX512())) {
6962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
6963 }
6964 return 0;
6965}
6966
6967unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
6968 if ((Subtarget->hasDQI())) {
6969 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
6970 }
6971 return 0;
6972}
6973
6974unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
6975switch (RetVT.SimpleTy) {
6976 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
6977 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
6978 default: return 0;
6979}
6980}
6981
6982unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0) {
6983 switch (VT.SimpleTy) {
6984 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
6985 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
6986 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
6987 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
6988 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
6989 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
6990 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
6991 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
6992 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
6993 default: return 0;
6994 }
6995}
6996
6997// FastEmit functions for X86ISD::STRICT_CVTTP2UI.
6998
6999unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
7000 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7001 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
7002 }
7003 return 0;
7004}
7005
7006unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
7007 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7008 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
7009 }
7010 return 0;
7011}
7012
7013unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
7014 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7015 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
7016 }
7017 return 0;
7018}
7019
7020unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
7021 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7022 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
7023 }
7024 return 0;
7025}
7026
7027unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
7028 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7029 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
7030 }
7031 return 0;
7032}
7033
7034unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
7035 if ((Subtarget->hasFP16())) {
7036 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
7037 }
7038 return 0;
7039}
7040
7041unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
7042switch (RetVT.SimpleTy) {
7043 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
7044 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
7045 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
7046 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
7047 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
7048 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
7049 default: return 0;
7050}
7051}
7052
7053unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
7054 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7055 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
7056 }
7057 return 0;
7058}
7059
7060unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
7061 if ((Subtarget->hasFP16())) {
7062 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
7063 }
7064 return 0;
7065}
7066
7067unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
7068switch (RetVT.SimpleTy) {
7069 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
7070 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
7071 default: return 0;
7072}
7073}
7074
7075unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
7076 if (RetVT.SimpleTy != MVT::v32i16)
7077 return 0;
7078 if ((Subtarget->hasFP16())) {
7079 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
7080 }
7081 return 0;
7082}
7083
7084unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
7085 if ((Subtarget->hasVLX())) {
7086 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
7087 }
7088 return 0;
7089}
7090
7091unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
7092 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7093 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
7094 }
7095 return 0;
7096}
7097
7098unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
7099 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7100 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
7101 }
7102 return 0;
7103}
7104
7105unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7106switch (RetVT.SimpleTy) {
7107 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
7108 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
7109 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
7110 default: return 0;
7111}
7112}
7113
7114unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
7115 if ((Subtarget->hasVLX())) {
7116 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
7117 }
7118 return 0;
7119}
7120
7121unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
7122 if ((Subtarget->hasDQI())) {
7123 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
7124 }
7125 return 0;
7126}
7127
7128unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
7129switch (RetVT.SimpleTy) {
7130 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
7131 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
7132 default: return 0;
7133}
7134}
7135
7136unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
7137 if (RetVT.SimpleTy != MVT::v16i32)
7138 return 0;
7139 if ((Subtarget->hasAVX512())) {
7140 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
7141 }
7142 return 0;
7143}
7144
7145unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
7146 if ((Subtarget->hasVLX())) {
7147 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
7148 }
7149 return 0;
7150}
7151
7152unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
7153 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7154 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
7155 }
7156 return 0;
7157}
7158
7159unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
7160switch (RetVT.SimpleTy) {
7161 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
7162 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
7163 default: return 0;
7164}
7165}
7166
7167unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
7168 if ((Subtarget->hasVLX())) {
7169 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
7170 }
7171 return 0;
7172}
7173
7174unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
7175 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7176 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
7177 }
7178 return 0;
7179}
7180
7181unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
7182switch (RetVT.SimpleTy) {
7183 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
7184 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
7185 default: return 0;
7186}
7187}
7188
7189unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
7190 if ((Subtarget->hasAVX512())) {
7191 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
7192 }
7193 return 0;
7194}
7195
7196unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
7197 if ((Subtarget->hasDQI())) {
7198 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
7199 }
7200 return 0;
7201}
7202
7203unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
7204switch (RetVT.SimpleTy) {
7205 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
7206 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
7207 default: return 0;
7208}
7209}
7210
7211unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0) {
7212 switch (VT.SimpleTy) {
7213 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
7214 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
7215 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
7216 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
7217 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
7218 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
7219 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
7220 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
7221 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
7222 default: return 0;
7223 }
7224}
7225
7226// FastEmit functions for X86ISD::STRICT_CVTUI2P.
7227
7228unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
7229 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7230 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
7231 }
7232 return 0;
7233}
7234
7235unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
7236 if ((Subtarget->hasVLX())) {
7237 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
7238 }
7239 return 0;
7240}
7241
7242unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
7243switch (RetVT.SimpleTy) {
7244 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
7245 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
7246 default: return 0;
7247}
7248}
7249
7250unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
7251 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7252 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
7253 }
7254 return 0;
7255}
7256
7257unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
7258 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7259 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
7260 }
7261 return 0;
7262}
7263
7264unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
7265switch (RetVT.SimpleTy) {
7266 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
7267 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
7268 default: return 0;
7269}
7270}
7271
7272unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
7273 if (RetVT.SimpleTy != MVT::v8f16)
7274 return 0;
7275 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7276 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
7277 }
7278 return 0;
7279}
7280
7281unsigned fastEmit_X86ISD_STRICT_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0) {
7282 switch (VT.SimpleTy) {
7283 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
7284 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
7285 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
7286 default: return 0;
7287 }
7288}
7289
7290// FastEmit functions for X86ISD::STRICT_VFPEXT.
7291
7292unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(unsigned Op0) {
7293 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7294 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
7295 }
7296 return 0;
7297}
7298
7299unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(unsigned Op0) {
7300 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7301 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
7302 }
7303 return 0;
7304}
7305
7306unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(unsigned Op0) {
7307 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7308 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
7309 }
7310 return 0;
7311}
7312
7313unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
7314switch (RetVT.SimpleTy) {
7315 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
7316 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
7317 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
7318 default: return 0;
7319}
7320}
7321
7322unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7323 if (RetVT.SimpleTy != MVT::v2f64)
7324 return 0;
7325 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7326 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
7327 }
7328 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7329 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
7330 }
7331 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7332 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
7333 }
7334 return 0;
7335}
7336
7337unsigned fastEmit_X86ISD_STRICT_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0) {
7338 switch (VT.SimpleTy) {
7339 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(RetVT, Op0);
7340 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(RetVT, Op0);
7341 default: return 0;
7342 }
7343}
7344
7345// FastEmit functions for X86ISD::STRICT_VFPROUND.
7346
7347unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7348 if (RetVT.SimpleTy != MVT::v8f16)
7349 return 0;
7350 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7351 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
7352 }
7353 return 0;
7354}
7355
7356unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
7357 if (RetVT.SimpleTy != MVT::v8f16)
7358 return 0;
7359 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7360 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
7361 }
7362 return 0;
7363}
7364
7365unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
7366 if (RetVT.SimpleTy != MVT::v16f16)
7367 return 0;
7368 if ((Subtarget->hasFP16())) {
7369 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
7370 }
7371 return 0;
7372}
7373
7374unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(unsigned Op0) {
7375 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7376 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
7377 }
7378 return 0;
7379}
7380
7381unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(unsigned Op0) {
7382 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7383 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
7384 }
7385 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7386 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
7387 }
7388 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7389 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
7390 }
7391 return 0;
7392}
7393
7394unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
7395switch (RetVT.SimpleTy) {
7396 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
7397 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
7398 default: return 0;
7399}
7400}
7401
7402unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(unsigned Op0) {
7403 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7404 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
7405 }
7406 return 0;
7407}
7408
7409unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(unsigned Op0) {
7410 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7411 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
7412 }
7413 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
7415 }
7416 return 0;
7417}
7418
7419unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
7420switch (RetVT.SimpleTy) {
7421 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
7422 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
7423 default: return 0;
7424}
7425}
7426
7427unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(unsigned Op0) {
7428 if ((Subtarget->hasFP16())) {
7429 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
7430 }
7431 return 0;
7432}
7433
7434unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(unsigned Op0) {
7435 if ((Subtarget->hasAVX512())) {
7436 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
7437 }
7438 return 0;
7439}
7440
7441unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
7442switch (RetVT.SimpleTy) {
7443 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
7444 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
7445 default: return 0;
7446}
7447}
7448
7449unsigned fastEmit_X86ISD_STRICT_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
7450 switch (VT.SimpleTy) {
7451 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(RetVT, Op0);
7452 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(RetVT, Op0);
7453 case MVT::v16f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(RetVT, Op0);
7454 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(RetVT, Op0);
7455 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(RetVT, Op0);
7456 case MVT::v8f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(RetVT, Op0);
7457 default: return 0;
7458 }
7459}
7460
7461// FastEmit functions for X86ISD::VBROADCAST.
7462
7463unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0) {
7464 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7465 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ128rr, RC: &X86::VR128XRegClass, Op0);
7466 }
7467 return 0;
7468}
7469
7470unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0) {
7471 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7472 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ256rr, RC: &X86::VR256XRegClass, Op0);
7473 }
7474 return 0;
7475}
7476
7477unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0) {
7478 if ((Subtarget->hasAVX512())) {
7479 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZrr, RC: &X86::VR512RegClass, Op0);
7480 }
7481 return 0;
7482}
7483
7484unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0) {
7485switch (RetVT.SimpleTy) {
7486 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0);
7487 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0);
7488 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0);
7489 default: return 0;
7490}
7491}
7492
7493unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0) {
7494 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7495 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ128rr, RC: &X86::VR128XRegClass, Op0);
7496 }
7497 return 0;
7498}
7499
7500unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0) {
7501 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7502 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ256rr, RC: &X86::VR256XRegClass, Op0);
7503 }
7504 return 0;
7505}
7506
7507unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0) {
7508 if ((Subtarget->hasAVX512())) {
7509 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZrr, RC: &X86::VR512RegClass, Op0);
7510 }
7511 return 0;
7512}
7513
7514unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0) {
7515switch (RetVT.SimpleTy) {
7516 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0);
7517 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0);
7518 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0);
7519 default: return 0;
7520}
7521}
7522
7523unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0) {
7524 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7525 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ128rr, RC: &X86::VR128XRegClass, Op0);
7526 }
7527 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7528 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBrr, RC: &X86::VR128RegClass, Op0);
7529 }
7530 return 0;
7531}
7532
7533unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0) {
7534 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7535 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ256rr, RC: &X86::VR256XRegClass, Op0);
7536 }
7537 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7538 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBYrr, RC: &X86::VR256RegClass, Op0);
7539 }
7540 return 0;
7541}
7542
7543unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0) {
7544 if ((Subtarget->hasBWI())) {
7545 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZrr, RC: &X86::VR512RegClass, Op0);
7546 }
7547 return 0;
7548}
7549
7550unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
7551switch (RetVT.SimpleTy) {
7552 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0);
7553 case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0);
7554 case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0);
7555 default: return 0;
7556}
7557}
7558
7559unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0) {
7560 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7561 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
7562 }
7563 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7564 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
7565 }
7566 return 0;
7567}
7568
7569unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0) {
7570 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7571 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
7572 }
7573 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7574 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
7575 }
7576 return 0;
7577}
7578
7579unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0) {
7580 if ((Subtarget->hasBWI())) {
7581 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
7582 }
7583 return 0;
7584}
7585
7586unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
7587switch (RetVT.SimpleTy) {
7588 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0);
7589 case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0);
7590 case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0);
7591 default: return 0;
7592}
7593}
7594
7595unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0) {
7596 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7597 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ128rr, RC: &X86::VR128XRegClass, Op0);
7598 }
7599 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7600 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrr, RC: &X86::VR128RegClass, Op0);
7601 }
7602 return 0;
7603}
7604
7605unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0) {
7606 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7607 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ256rr, RC: &X86::VR256XRegClass, Op0);
7608 }
7609 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7610 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDYrr, RC: &X86::VR256RegClass, Op0);
7611 }
7612 return 0;
7613}
7614
7615unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0) {
7616 if ((Subtarget->hasAVX512())) {
7617 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZrr, RC: &X86::VR512RegClass, Op0);
7618 }
7619 return 0;
7620}
7621
7622unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
7623switch (RetVT.SimpleTy) {
7624 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0);
7625 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0);
7626 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0);
7627 default: return 0;
7628}
7629}
7630
7631unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0) {
7632 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7633 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ128rr, RC: &X86::VR128XRegClass, Op0);
7634 }
7635 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7636 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrr, RC: &X86::VR128RegClass, Op0);
7637 }
7638 return 0;
7639}
7640
7641unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0) {
7642 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7643 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ256rr, RC: &X86::VR256XRegClass, Op0);
7644 }
7645 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7646 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQYrr, RC: &X86::VR256RegClass, Op0);
7647 }
7648 return 0;
7649}
7650
7651unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0) {
7652 if ((Subtarget->hasAVX512())) {
7653 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZrr, RC: &X86::VR512RegClass, Op0);
7654 }
7655 return 0;
7656}
7657
7658unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
7659switch (RetVT.SimpleTy) {
7660 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0);
7661 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0);
7662 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0);
7663 default: return 0;
7664}
7665}
7666
7667unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(unsigned Op0) {
7668 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7669 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
7670 }
7671 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7672 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
7673 }
7674 return 0;
7675}
7676
7677unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(unsigned Op0) {
7678 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7679 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
7680 }
7681 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7682 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
7683 }
7684 return 0;
7685}
7686
7687unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(unsigned Op0) {
7688 if ((Subtarget->hasBWI())) {
7689 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
7690 }
7691 return 0;
7692}
7693
7694unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
7695switch (RetVT.SimpleTy) {
7696 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Op0);
7697 case MVT::v16f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Op0);
7698 case MVT::v32f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Op0);
7699 default: return 0;
7700}
7701}
7702
7703unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(unsigned Op0) {
7704 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
7705 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
7706 }
7707 return 0;
7708}
7709
7710unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(unsigned Op0) {
7711 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
7712 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
7713 }
7714 return 0;
7715}
7716
7717unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(unsigned Op0) {
7718 if ((Subtarget->hasBF16())) {
7719 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
7720 }
7721 return 0;
7722}
7723
7724unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
7725switch (RetVT.SimpleTy) {
7726 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Op0);
7727 case MVT::v16bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Op0);
7728 case MVT::v32bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Op0);
7729 default: return 0;
7730}
7731}
7732
7733unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0) {
7734 if ((Subtarget->hasVLX())) {
7735 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ128rr, RC: &X86::VR128XRegClass, Op0);
7736 }
7737 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7738 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSrr, RC: &X86::VR128RegClass, Op0);
7739 }
7740 return 0;
7741}
7742
7743unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0) {
7744 if ((Subtarget->hasVLX())) {
7745 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ256rr, RC: &X86::VR256XRegClass, Op0);
7746 }
7747 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7748 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSYrr, RC: &X86::VR256RegClass, Op0);
7749 }
7750 return 0;
7751}
7752
7753unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0) {
7754 if ((Subtarget->hasAVX512())) {
7755 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZrr, RC: &X86::VR512RegClass, Op0);
7756 }
7757 return 0;
7758}
7759
7760unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7761switch (RetVT.SimpleTy) {
7762 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0);
7763 case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0);
7764 case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0);
7765 default: return 0;
7766}
7767}
7768
7769unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0) {
7770 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7771 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7772 }
7773 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7774 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7775 }
7776 return 0;
7777}
7778
7779unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0) {
7780 if ((Subtarget->hasVLX())) {
7781 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZ256rr, RC: &X86::VR256XRegClass, Op0);
7782 }
7783 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7784 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDYrr, RC: &X86::VR256RegClass, Op0);
7785 }
7786 return 0;
7787}
7788
7789unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0) {
7790 if ((Subtarget->hasAVX512())) {
7791 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZrr, RC: &X86::VR512RegClass, Op0);
7792 }
7793 return 0;
7794}
7795
7796unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
7797switch (RetVT.SimpleTy) {
7798 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0);
7799 case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0);
7800 case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0);
7801 default: return 0;
7802}
7803}
7804
7805unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0) {
7806 switch (VT.SimpleTy) {
7807 case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0);
7808 case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0);
7809 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0);
7810 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0);
7811 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0);
7812 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0);
7813 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(RetVT, Op0);
7814 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(RetVT, Op0);
7815 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0);
7816 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0);
7817 default: return 0;
7818 }
7819}
7820
7821// FastEmit functions for X86ISD::VBROADCASTM.
7822
7823unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0) {
7824 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
7825 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ128rr, RC: &X86::VR128XRegClass, Op0);
7826 }
7827 return 0;
7828}
7829
7830unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0) {
7831 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
7832 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ256rr, RC: &X86::VR256XRegClass, Op0);
7833 }
7834 return 0;
7835}
7836
7837unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0) {
7838 if ((Subtarget->hasCDI())) {
7839 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZrr, RC: &X86::VR512RegClass, Op0);
7840 }
7841 return 0;
7842}
7843
7844unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0) {
7845switch (RetVT.SimpleTy) {
7846 case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0);
7847 case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0);
7848 case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0);
7849 default: return 0;
7850}
7851}
7852
7853unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0) {
7854 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
7855 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ128rr, RC: &X86::VR128XRegClass, Op0);
7856 }
7857 return 0;
7858}
7859
7860unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0) {
7861 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
7862 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ256rr, RC: &X86::VR256XRegClass, Op0);
7863 }
7864 return 0;
7865}
7866
7867unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0) {
7868 if ((Subtarget->hasCDI())) {
7869 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZrr, RC: &X86::VR512RegClass, Op0);
7870 }
7871 return 0;
7872}
7873
7874unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0) {
7875switch (RetVT.SimpleTy) {
7876 case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0);
7877 case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0);
7878 case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0);
7879 default: return 0;
7880}
7881}
7882
7883unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0) {
7884 switch (VT.SimpleTy) {
7885 case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0);
7886 case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0);
7887 default: return 0;
7888 }
7889}
7890
7891// FastEmit functions for X86ISD::VFPEXT.
7892
7893unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(unsigned Op0) {
7894 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7895 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
7896 }
7897 return 0;
7898}
7899
7900unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(unsigned Op0) {
7901 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7902 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
7903 }
7904 return 0;
7905}
7906
7907unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(unsigned Op0) {
7908 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7909 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
7910 }
7911 return 0;
7912}
7913
7914unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
7915switch (RetVT.SimpleTy) {
7916 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
7917 case MVT::v2f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
7918 case MVT::v4f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
7919 default: return 0;
7920}
7921}
7922
7923unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7924 if (RetVT.SimpleTy != MVT::v2f64)
7925 return 0;
7926 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7927 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
7928 }
7929 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7930 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
7931 }
7932 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7933 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
7934 }
7935 return 0;
7936}
7937
7938unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0) {
7939 switch (VT.SimpleTy) {
7940 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(RetVT, Op0);
7941 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0);
7942 default: return 0;
7943 }
7944}
7945
7946// FastEmit functions for X86ISD::VFPEXT_SAE.
7947
7948unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
7949 if (RetVT.SimpleTy != MVT::v8f64)
7950 return 0;
7951 if ((Subtarget->hasFP16())) {
7952 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrrb, RC: &X86::VR512RegClass, Op0);
7953 }
7954 return 0;
7955}
7956
7957unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
7958 if (RetVT.SimpleTy != MVT::v16f32)
7959 return 0;
7960 if ((Subtarget->hasFP16())) {
7961 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrrb, RC: &X86::VR512RegClass, Op0);
7962 }
7963 return 0;
7964}
7965
7966unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
7967 if (RetVT.SimpleTy != MVT::v8f64)
7968 return 0;
7969 if ((Subtarget->hasAVX512())) {
7970 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrrb, RC: &X86::VR512RegClass, Op0);
7971 }
7972 return 0;
7973}
7974
7975unsigned fastEmit_X86ISD_VFPEXT_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
7976 switch (VT.SimpleTy) {
7977 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(RetVT, Op0);
7978 case MVT::v16f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(RetVT, Op0);
7979 case MVT::v8f32: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(RetVT, Op0);
7980 default: return 0;
7981 }
7982}
7983
7984// FastEmit functions for X86ISD::VFPROUND.
7985
7986unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7987 if (RetVT.SimpleTy != MVT::v8f16)
7988 return 0;
7989 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7990 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
7991 }
7992 return 0;
7993}
7994
7995unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(unsigned Op0) {
7996 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7997 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
7998 }
7999 return 0;
8000}
8001
8002unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(unsigned Op0) {
8003 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
8004 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
8005 }
8006 if ((Subtarget->hasAVXNECONVERT())) {
8007 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Yrr, RC: &X86::VR128RegClass, Op0);
8008 }
8009 return 0;
8010}
8011
8012unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
8013switch (RetVT.SimpleTy) {
8014 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Op0);
8015 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Op0);
8016 default: return 0;
8017}
8018}
8019
8020unsigned fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(unsigned Op0) {
8021 if ((Subtarget->hasFP16())) {
8022 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
8023 }
8024 return 0;
8025}
8026
8027unsigned fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(unsigned Op0) {
8028 if ((Subtarget->hasBF16())) {
8029 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
8030 }
8031 return 0;
8032}
8033
8034unsigned fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
8035switch (RetVT.SimpleTy) {
8036 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Op0);
8037 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Op0);
8038 default: return 0;
8039}
8040}
8041
8042unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(unsigned Op0) {
8043 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8044 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8045 }
8046 return 0;
8047}
8048
8049unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(unsigned Op0) {
8050 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8051 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8052 }
8053 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8054 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8055 }
8056 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8057 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8058 }
8059 return 0;
8060}
8061
8062unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
8063switch (RetVT.SimpleTy) {
8064 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
8065 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
8066 default: return 0;
8067}
8068}
8069
8070unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(unsigned Op0) {
8071 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8072 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8073 }
8074 return 0;
8075}
8076
8077unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(unsigned Op0) {
8078 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8079 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
8080 }
8081 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8082 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
8083 }
8084 return 0;
8085}
8086
8087unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
8088switch (RetVT.SimpleTy) {
8089 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
8090 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
8091 default: return 0;
8092}
8093}
8094
8095unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(unsigned Op0) {
8096 if ((Subtarget->hasFP16())) {
8097 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
8098 }
8099 return 0;
8100}
8101
8102unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(unsigned Op0) {
8103 if ((Subtarget->hasAVX512())) {
8104 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
8105 }
8106 return 0;
8107}
8108
8109unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
8110switch (RetVT.SimpleTy) {
8111 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
8112 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
8113 default: return 0;
8114}
8115}
8116
8117unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
8118 switch (VT.SimpleTy) {
8119 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(RetVT, Op0);
8120 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(RetVT, Op0);
8121 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(RetVT, Op0);
8122 case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0);
8123 case MVT::v4f64: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(RetVT, Op0);
8124 case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(RetVT, Op0);
8125 default: return 0;
8126 }
8127}
8128
8129// FastEmit functions for X86ISD::VTRUNC.
8130
8131unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
8132 if (RetVT.SimpleTy != MVT::v16i8)
8133 return 0;
8134 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8135 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ128rr, RC: &X86::VR128XRegClass, Op0);
8136 }
8137 return 0;
8138}
8139
8140unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0) {
8141 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8142 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ128rr, RC: &X86::VR128XRegClass, Op0);
8143 }
8144 return 0;
8145}
8146
8147unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0) {
8148 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8149 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ128rr, RC: &X86::VR128XRegClass, Op0);
8150 }
8151 return 0;
8152}
8153
8154unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
8155switch (RetVT.SimpleTy) {
8156 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0);
8157 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0);
8158 default: return 0;
8159}
8160}
8161
8162unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
8163 if (RetVT.SimpleTy != MVT::v16i8)
8164 return 0;
8165 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8166 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ256rr, RC: &X86::VR128XRegClass, Op0);
8167 }
8168 return 0;
8169}
8170
8171unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0) {
8172 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8173 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ128rr, RC: &X86::VR128XRegClass, Op0);
8174 }
8175 return 0;
8176}
8177
8178unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0) {
8179 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8180 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ128rr, RC: &X86::VR128XRegClass, Op0);
8181 }
8182 return 0;
8183}
8184
8185unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0) {
8186 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8187 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ128rr, RC: &X86::VR128XRegClass, Op0);
8188 }
8189 return 0;
8190}
8191
8192unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
8193switch (RetVT.SimpleTy) {
8194 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0);
8195 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0);
8196 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0);
8197 default: return 0;
8198}
8199}
8200
8201unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0) {
8202 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8203 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ256rr, RC: &X86::VR128XRegClass, Op0);
8204 }
8205 return 0;
8206}
8207
8208unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0) {
8209 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8210 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ256rr, RC: &X86::VR128XRegClass, Op0);
8211 }
8212 return 0;
8213}
8214
8215unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
8216switch (RetVT.SimpleTy) {
8217 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0);
8218 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0);
8219 default: return 0;
8220}
8221}
8222
8223unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
8224 if (RetVT.SimpleTy != MVT::v16i8)
8225 return 0;
8226 if ((Subtarget->hasAVX512())) {
8227 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZrr, RC: &X86::VR128XRegClass, Op0);
8228 }
8229 return 0;
8230}
8231
8232unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0) {
8233 switch (VT.SimpleTy) {
8234 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0);
8235 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0);
8236 case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0);
8237 case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0);
8238 case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0);
8239 case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0);
8240 default: return 0;
8241 }
8242}
8243
8244// FastEmit functions for X86ISD::VTRUNCS.
8245
8246unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
8247 if (RetVT.SimpleTy != MVT::v16i8)
8248 return 0;
8249 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8250 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
8251 }
8252 return 0;
8253}
8254
8255unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
8256 if (RetVT.SimpleTy != MVT::v16i8)
8257 return 0;
8258 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8259 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
8260 }
8261 return 0;
8262}
8263
8264unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
8265 if (RetVT.SimpleTy != MVT::v32i8)
8266 return 0;
8267 if ((Subtarget->hasBWI())) {
8268 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZrr, RC: &X86::VR256XRegClass, Op0);
8269 }
8270 return 0;
8271}
8272
8273unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0) {
8274 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8275 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
8276 }
8277 return 0;
8278}
8279
8280unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0) {
8281 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8282 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
8283 }
8284 return 0;
8285}
8286
8287unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
8288switch (RetVT.SimpleTy) {
8289 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0);
8290 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0);
8291 default: return 0;
8292}
8293}
8294
8295unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0) {
8296 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8297 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
8298 }
8299 return 0;
8300}
8301
8302unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0) {
8303 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8304 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
8305 }
8306 return 0;
8307}
8308
8309unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
8310switch (RetVT.SimpleTy) {
8311 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0);
8312 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0);
8313 default: return 0;
8314}
8315}
8316
8317unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0) {
8318 if ((Subtarget->hasAVX512())) {
8319 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZrr, RC: &X86::VR128XRegClass, Op0);
8320 }
8321 return 0;
8322}
8323
8324unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0) {
8325 if ((Subtarget->hasAVX512())) {
8326 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZrr, RC: &X86::VR256XRegClass, Op0);
8327 }
8328 return 0;
8329}
8330
8331unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
8332switch (RetVT.SimpleTy) {
8333 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0);
8334 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0);
8335 default: return 0;
8336}
8337}
8338
8339unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0) {
8340 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8341 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
8342 }
8343 return 0;
8344}
8345
8346unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0) {
8347 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8348 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
8349 }
8350 return 0;
8351}
8352
8353unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0) {
8354 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8355 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
8356 }
8357 return 0;
8358}
8359
8360unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
8361switch (RetVT.SimpleTy) {
8362 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0);
8363 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0);
8364 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0);
8365 default: return 0;
8366}
8367}
8368
8369unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0) {
8370 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8371 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
8372 }
8373 return 0;
8374}
8375
8376unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0) {
8377 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8378 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
8379 }
8380 return 0;
8381}
8382
8383unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0) {
8384 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8385 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
8386 }
8387 return 0;
8388}
8389
8390unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
8391switch (RetVT.SimpleTy) {
8392 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0);
8393 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0);
8394 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0);
8395 default: return 0;
8396}
8397}
8398
8399unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0) {
8400 if ((Subtarget->hasAVX512())) {
8401 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZrr, RC: &X86::VR128XRegClass, Op0);
8402 }
8403 return 0;
8404}
8405
8406unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0) {
8407 if ((Subtarget->hasAVX512())) {
8408 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZrr, RC: &X86::VR128XRegClass, Op0);
8409 }
8410 return 0;
8411}
8412
8413unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0) {
8414 if ((Subtarget->hasAVX512())) {
8415 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZrr, RC: &X86::VR256XRegClass, Op0);
8416 }
8417 return 0;
8418}
8419
8420unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
8421switch (RetVT.SimpleTy) {
8422 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0);
8423 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0);
8424 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0);
8425 default: return 0;
8426}
8427}
8428
8429unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0) {
8430 switch (VT.SimpleTy) {
8431 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0);
8432 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0);
8433 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0);
8434 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0);
8435 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0);
8436 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0);
8437 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0);
8438 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0);
8439 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0);
8440 default: return 0;
8441 }
8442}
8443
8444// FastEmit functions for X86ISD::VTRUNCUS.
8445
8446unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
8447 if (RetVT.SimpleTy != MVT::v16i8)
8448 return 0;
8449 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8450 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
8451 }
8452 return 0;
8453}
8454
8455unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
8456 if (RetVT.SimpleTy != MVT::v16i8)
8457 return 0;
8458 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8459 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
8460 }
8461 return 0;
8462}
8463
8464unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
8465 if (RetVT.SimpleTy != MVT::v32i8)
8466 return 0;
8467 if ((Subtarget->hasBWI())) {
8468 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZrr, RC: &X86::VR256XRegClass, Op0);
8469 }
8470 return 0;
8471}
8472
8473unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0) {
8474 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8475 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
8476 }
8477 return 0;
8478}
8479
8480unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0) {
8481 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8482 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
8483 }
8484 return 0;
8485}
8486
8487unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
8488switch (RetVT.SimpleTy) {
8489 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0);
8490 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0);
8491 default: return 0;
8492}
8493}
8494
8495unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0) {
8496 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8497 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
8498 }
8499 return 0;
8500}
8501
8502unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0) {
8503 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8504 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
8505 }
8506 return 0;
8507}
8508
8509unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
8510switch (RetVT.SimpleTy) {
8511 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0);
8512 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0);
8513 default: return 0;
8514}
8515}
8516
8517unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0) {
8518 if ((Subtarget->hasAVX512())) {
8519 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZrr, RC: &X86::VR128XRegClass, Op0);
8520 }
8521 return 0;
8522}
8523
8524unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0) {
8525 if ((Subtarget->hasAVX512())) {
8526 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZrr, RC: &X86::VR256XRegClass, Op0);
8527 }
8528 return 0;
8529}
8530
8531unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
8532switch (RetVT.SimpleTy) {
8533 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0);
8534 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0);
8535 default: return 0;
8536}
8537}
8538
8539unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0) {
8540 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8541 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
8542 }
8543 return 0;
8544}
8545
8546unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0) {
8547 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8548 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
8549 }
8550 return 0;
8551}
8552
8553unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0) {
8554 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8555 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
8556 }
8557 return 0;
8558}
8559
8560unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
8561switch (RetVT.SimpleTy) {
8562 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0);
8563 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0);
8564 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0);
8565 default: return 0;
8566}
8567}
8568
8569unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0) {
8570 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8571 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
8572 }
8573 return 0;
8574}
8575
8576unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0) {
8577 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8578 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
8579 }
8580 return 0;
8581}
8582
8583unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0) {
8584 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8585 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
8586 }
8587 return 0;
8588}
8589
8590unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
8591switch (RetVT.SimpleTy) {
8592 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0);
8593 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0);
8594 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0);
8595 default: return 0;
8596}
8597}
8598
8599unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0) {
8600 if ((Subtarget->hasAVX512())) {
8601 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZrr, RC: &X86::VR128XRegClass, Op0);
8602 }
8603 return 0;
8604}
8605
8606unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0) {
8607 if ((Subtarget->hasAVX512())) {
8608 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZrr, RC: &X86::VR128XRegClass, Op0);
8609 }
8610 return 0;
8611}
8612
8613unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0) {
8614 if ((Subtarget->hasAVX512())) {
8615 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZrr, RC: &X86::VR256XRegClass, Op0);
8616 }
8617 return 0;
8618}
8619
8620unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
8621switch (RetVT.SimpleTy) {
8622 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0);
8623 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0);
8624 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0);
8625 default: return 0;
8626}
8627}
8628
8629unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0) {
8630 switch (VT.SimpleTy) {
8631 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0);
8632 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0);
8633 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0);
8634 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0);
8635 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0);
8636 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0);
8637 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0);
8638 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0);
8639 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0);
8640 default: return 0;
8641 }
8642}
8643
8644// FastEmit functions for X86ISD::VZEXT_MOVL.
8645
8646unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
8647 if (RetVT.SimpleTy != MVT::v2i64)
8648 return 0;
8649 if ((Subtarget->hasAVX512())) {
8650 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
8651 }
8652 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8653 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
8654 }
8655 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
8656 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
8657 }
8658 return 0;
8659}
8660
8661unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
8662 if (RetVT.SimpleTy != MVT::v2f64)
8663 return 0;
8664 if ((Subtarget->hasAVX512())) {
8665 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
8666 }
8667 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8668 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
8669 }
8670 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
8671 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
8672 }
8673 return 0;
8674}
8675
8676unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0) {
8677 switch (VT.SimpleTy) {
8678 case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0);
8679 case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0);
8680 default: return 0;
8681 }
8682}
8683
8684// Top-level FastEmit function.
8685
8686unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0) override {
8687 switch (Opcode) {
8688 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
8689 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
8690 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
8691 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
8692 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
8693 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
8694 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
8695 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
8696 case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0);
8697 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
8698 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
8699 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
8700 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
8701 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
8702 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
8703 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
8704 case ISD::LLRINT: return fastEmit_ISD_LLRINT_r(VT, RetVT, Op0);
8705 case ISD::LRINT: return fastEmit_ISD_LRINT_r(VT, RetVT, Op0);
8706 case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0);
8707 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
8708 case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
8709 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
8710 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
8711 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
8712 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
8713 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
8714 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
8715 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
8716 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
8717 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
8718 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
8719 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
8720 case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
8721 case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0);
8722 case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0);
8723 case X86ISD::CVTNEPS2BF16: return fastEmit_X86ISD_CVTNEPS2BF16_r(VT, RetVT, Op0);
8724 case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0);
8725 case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0);
8726 case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0);
8727 case X86ISD::CVTPH2PS_SAE: return fastEmit_X86ISD_CVTPH2PS_SAE_r(VT, RetVT, Op0);
8728 case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0);
8729 case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0);
8730 case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0);
8731 case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0);
8732 case X86ISD::CVTTP2SI_SAE: return fastEmit_X86ISD_CVTTP2SI_SAE_r(VT, RetVT, Op0);
8733 case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0);
8734 case X86ISD::CVTTP2UI_SAE: return fastEmit_X86ISD_CVTTP2UI_SAE_r(VT, RetVT, Op0);
8735 case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0);
8736 case X86ISD::CVTTS2SI_SAE: return fastEmit_X86ISD_CVTTS2SI_SAE_r(VT, RetVT, Op0);
8737 case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0);
8738 case X86ISD::CVTTS2UI_SAE: return fastEmit_X86ISD_CVTTS2UI_SAE_r(VT, RetVT, Op0);
8739 case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0);
8740 case X86ISD::DYN_ALLOCA: return fastEmit_X86ISD_DYN_ALLOCA_r(VT, RetVT, Op0);
8741 case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0);
8742 case X86ISD::FGETEXP: return fastEmit_X86ISD_FGETEXP_r(VT, RetVT, Op0);
8743 case X86ISD::FGETEXP_SAE: return fastEmit_X86ISD_FGETEXP_SAE_r(VT, RetVT, Op0);
8744 case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0);
8745 case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0);
8746 case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0);
8747 case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0);
8748 case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0);
8749 case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0);
8750 case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0);
8751 case X86ISD::MOVQ2DQ: return fastEmit_X86ISD_MOVQ2DQ_r(VT, RetVT, Op0);
8752 case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0);
8753 case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0);
8754 case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0);
8755 case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0);
8756 case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0);
8757 case X86ISD::PROBED_ALLOCA: return fastEmit_X86ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
8758 case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0);
8759 case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0);
8760 case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0);
8761 case X86ISD::STRICT_CVTPH2PS: return fastEmit_X86ISD_STRICT_CVTPH2PS_r(VT, RetVT, Op0);
8762 case X86ISD::STRICT_CVTSI2P: return fastEmit_X86ISD_STRICT_CVTSI2P_r(VT, RetVT, Op0);
8763 case X86ISD::STRICT_CVTTP2SI: return fastEmit_X86ISD_STRICT_CVTTP2SI_r(VT, RetVT, Op0);
8764 case X86ISD::STRICT_CVTTP2UI: return fastEmit_X86ISD_STRICT_CVTTP2UI_r(VT, RetVT, Op0);
8765 case X86ISD::STRICT_CVTUI2P: return fastEmit_X86ISD_STRICT_CVTUI2P_r(VT, RetVT, Op0);
8766 case X86ISD::STRICT_VFPEXT: return fastEmit_X86ISD_STRICT_VFPEXT_r(VT, RetVT, Op0);
8767 case X86ISD::STRICT_VFPROUND: return fastEmit_X86ISD_STRICT_VFPROUND_r(VT, RetVT, Op0);
8768 case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0);
8769 case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0);
8770 case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0);
8771 case X86ISD::VFPEXT_SAE: return fastEmit_X86ISD_VFPEXT_SAE_r(VT, RetVT, Op0);
8772 case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0);
8773 case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0);
8774 case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0);
8775 case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0);
8776 case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0);
8777 default: return 0;
8778 }
8779}
8780
8781// FastEmit functions for ISD::ADD.
8782
8783unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8784 if (RetVT.SimpleTy != MVT::i8)
8785 return 0;
8786 if ((Subtarget->hasNDD())) {
8787 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
8788 }
8789 if ((!Subtarget->hasNDD())) {
8790 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr, RC: &X86::GR8RegClass, Op0, Op1);
8791 }
8792 return 0;
8793}
8794
8795unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8796 if (RetVT.SimpleTy != MVT::i16)
8797 return 0;
8798 if ((Subtarget->hasNDD())) {
8799 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
8800 }
8801 if ((!Subtarget->hasNDD())) {
8802 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr, RC: &X86::GR16RegClass, Op0, Op1);
8803 }
8804 return 0;
8805}
8806
8807unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8808 if (RetVT.SimpleTy != MVT::i32)
8809 return 0;
8810 if ((Subtarget->hasNDD())) {
8811 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
8812 }
8813 if ((!Subtarget->hasNDD())) {
8814 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr, RC: &X86::GR32RegClass, Op0, Op1);
8815 }
8816 return 0;
8817}
8818
8819unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8820 if (RetVT.SimpleTy != MVT::i64)
8821 return 0;
8822 if ((Subtarget->hasNDD())) {
8823 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
8824 }
8825 if ((!Subtarget->hasNDD())) {
8826 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr, RC: &X86::GR64RegClass, Op0, Op1);
8827 }
8828 return 0;
8829}
8830
8831unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8832 if (RetVT.SimpleTy != MVT::v16i8)
8833 return 0;
8834 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8835 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
8836 }
8837 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8838 return fastEmitInst_rr(MachineInstOpcode: X86::PADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
8839 }
8840 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8841 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
8842 }
8843 return 0;
8844}
8845
8846unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8847 if (RetVT.SimpleTy != MVT::v32i8)
8848 return 0;
8849 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8850 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
8851 }
8852 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8853 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBYrr, RC: &X86::VR256RegClass, Op0, Op1);
8854 }
8855 return 0;
8856}
8857
8858unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8859 if (RetVT.SimpleTy != MVT::v64i8)
8860 return 0;
8861 if ((Subtarget->hasBWI())) {
8862 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZrr, RC: &X86::VR512RegClass, Op0, Op1);
8863 }
8864 return 0;
8865}
8866
8867unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8868 if (RetVT.SimpleTy != MVT::v8i16)
8869 return 0;
8870 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8871 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
8872 }
8873 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8874 return fastEmitInst_rr(MachineInstOpcode: X86::PADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
8875 }
8876 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8877 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
8878 }
8879 return 0;
8880}
8881
8882unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8883 if (RetVT.SimpleTy != MVT::v16i16)
8884 return 0;
8885 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8886 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
8887 }
8888 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8889 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
8890 }
8891 return 0;
8892}
8893
8894unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8895 if (RetVT.SimpleTy != MVT::v32i16)
8896 return 0;
8897 if ((Subtarget->hasBWI())) {
8898 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
8899 }
8900 return 0;
8901}
8902
8903unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8904 if (RetVT.SimpleTy != MVT::v4i32)
8905 return 0;
8906 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8907 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
8908 }
8909 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8910 return fastEmitInst_rr(MachineInstOpcode: X86::PADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
8911 }
8912 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8913 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
8914 }
8915 return 0;
8916}
8917
8918unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8919 if (RetVT.SimpleTy != MVT::v8i32)
8920 return 0;
8921 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8922 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
8923 }
8924 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8925 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
8926 }
8927 return 0;
8928}
8929
8930unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8931 if (RetVT.SimpleTy != MVT::v16i32)
8932 return 0;
8933 if ((Subtarget->hasAVX512())) {
8934 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
8935 }
8936 return 0;
8937}
8938
8939unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8940 if (RetVT.SimpleTy != MVT::v2i64)
8941 return 0;
8942 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8943 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
8944 }
8945 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8946 return fastEmitInst_rr(MachineInstOpcode: X86::PADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
8947 }
8948 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8949 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
8950 }
8951 return 0;
8952}
8953
8954unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8955 if (RetVT.SimpleTy != MVT::v4i64)
8956 return 0;
8957 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8958 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
8959 }
8960 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8961 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
8962 }
8963 return 0;
8964}
8965
8966unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8967 if (RetVT.SimpleTy != MVT::v8i64)
8968 return 0;
8969 if ((Subtarget->hasAVX512())) {
8970 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
8971 }
8972 return 0;
8973}
8974
8975unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8976 switch (VT.SimpleTy) {
8977 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op1);
8978 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op1);
8979 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
8980 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
8981 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
8982 case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op1);
8983 case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op1);
8984 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
8985 case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op1);
8986 case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op1);
8987 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
8988 case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op1);
8989 case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op1);
8990 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
8991 case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op1);
8992 case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op1);
8993 default: return 0;
8994 }
8995}
8996
8997// FastEmit functions for ISD::AND.
8998
8999unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9000 if (RetVT.SimpleTy != MVT::i8)
9001 return 0;
9002 if ((Subtarget->hasNDD())) {
9003 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
9004 }
9005 if ((!Subtarget->hasNDD())) {
9006 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr, RC: &X86::GR8RegClass, Op0, Op1);
9007 }
9008 return 0;
9009}
9010
9011unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9012 if (RetVT.SimpleTy != MVT::i16)
9013 return 0;
9014 if ((Subtarget->hasNDD())) {
9015 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
9016 }
9017 if ((!Subtarget->hasNDD())) {
9018 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr, RC: &X86::GR16RegClass, Op0, Op1);
9019 }
9020 return 0;
9021}
9022
9023unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9024 if (RetVT.SimpleTy != MVT::i32)
9025 return 0;
9026 if ((Subtarget->hasNDD())) {
9027 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
9028 }
9029 if ((!Subtarget->hasNDD())) {
9030 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr, RC: &X86::GR32RegClass, Op0, Op1);
9031 }
9032 return 0;
9033}
9034
9035unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9036 if (RetVT.SimpleTy != MVT::i64)
9037 return 0;
9038 if ((Subtarget->hasNDD())) {
9039 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
9040 }
9041 if ((!Subtarget->hasNDD())) {
9042 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr, RC: &X86::GR64RegClass, Op0, Op1);
9043 }
9044 return 0;
9045}
9046
9047unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9048 if (RetVT.SimpleTy != MVT::v8i1)
9049 return 0;
9050 if ((Subtarget->hasDQI())) {
9051 return fastEmitInst_rr(MachineInstOpcode: X86::KANDBrr, RC: &X86::VK8RegClass, Op0, Op1);
9052 }
9053 return 0;
9054}
9055
9056unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9057 if (RetVT.SimpleTy != MVT::v16i1)
9058 return 0;
9059 if ((Subtarget->hasAVX512())) {
9060 return fastEmitInst_rr(MachineInstOpcode: X86::KANDWrr, RC: &X86::VK16RegClass, Op0, Op1);
9061 }
9062 return 0;
9063}
9064
9065unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9066 if (RetVT.SimpleTy != MVT::v32i1)
9067 return 0;
9068 if ((Subtarget->hasBWI())) {
9069 return fastEmitInst_rr(MachineInstOpcode: X86::KANDDrr, RC: &X86::VK32RegClass, Op0, Op1);
9070 }
9071 return 0;
9072}
9073
9074unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9075 if (RetVT.SimpleTy != MVT::v64i1)
9076 return 0;
9077 if ((Subtarget->hasBWI())) {
9078 return fastEmitInst_rr(MachineInstOpcode: X86::KANDQrr, RC: &X86::VK64RegClass, Op0, Op1);
9079 }
9080 return 0;
9081}
9082
9083unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9084 if (RetVT.SimpleTy != MVT::v16i8)
9085 return 0;
9086 if ((Subtarget->hasVLX())) {
9087 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9088 }
9089 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9090 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
9091 }
9092 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9093 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
9094 }
9095 return 0;
9096}
9097
9098unsigned fastEmit_ISD_AND_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9099 if (RetVT.SimpleTy != MVT::v32i8)
9100 return 0;
9101 if ((Subtarget->hasVLX())) {
9102 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9103 }
9104 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
9105 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
9106 }
9107 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9108 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
9109 }
9110 return 0;
9111}
9112
9113unsigned fastEmit_ISD_AND_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9114 if (RetVT.SimpleTy != MVT::v64i8)
9115 return 0;
9116 if ((Subtarget->hasAVX512())) {
9117 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
9118 }
9119 return 0;
9120}
9121
9122unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9123 if (RetVT.SimpleTy != MVT::v8i16)
9124 return 0;
9125 if ((Subtarget->hasVLX())) {
9126 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9127 }
9128 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9129 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
9130 }
9131 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9132 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
9133 }
9134 return 0;
9135}
9136
9137unsigned fastEmit_ISD_AND_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9138 if (RetVT.SimpleTy != MVT::v16i16)
9139 return 0;
9140 if ((Subtarget->hasVLX())) {
9141 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9142 }
9143 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
9144 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
9145 }
9146 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9147 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
9148 }
9149 return 0;
9150}
9151
9152unsigned fastEmit_ISD_AND_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9153 if (RetVT.SimpleTy != MVT::v32i16)
9154 return 0;
9155 if ((Subtarget->hasAVX512())) {
9156 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
9157 }
9158 return 0;
9159}
9160
9161unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9162 if (RetVT.SimpleTy != MVT::v4i32)
9163 return 0;
9164 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9165 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
9166 }
9167 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9168 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
9169 }
9170 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9171 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9172 }
9173 return 0;
9174}
9175
9176unsigned fastEmit_ISD_AND_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9177 if (RetVT.SimpleTy != MVT::v8i32)
9178 return 0;
9179 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
9180 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
9181 }
9182 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9183 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
9184 }
9185 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9186 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9187 }
9188 return 0;
9189}
9190
9191unsigned fastEmit_ISD_AND_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9192 if (RetVT.SimpleTy != MVT::v16i32)
9193 return 0;
9194 if ((Subtarget->hasAVX512())) {
9195 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
9196 }
9197 return 0;
9198}
9199
9200unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9201 if (RetVT.SimpleTy != MVT::v2i64)
9202 return 0;
9203 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9204 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9205 }
9206 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9207 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
9208 }
9209 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9210 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
9211 }
9212 return 0;
9213}
9214
9215unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9216 if (RetVT.SimpleTy != MVT::v4i64)
9217 return 0;
9218 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
9219 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
9220 }
9221 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9222 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9223 }
9224 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9225 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
9226 }
9227 return 0;
9228}
9229
9230unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9231 if (RetVT.SimpleTy != MVT::v8i64)
9232 return 0;
9233 if ((Subtarget->hasAVX512())) {
9234 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
9235 }
9236 return 0;
9237}
9238
9239unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9240 switch (VT.SimpleTy) {
9241 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op1);
9242 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op1);
9243 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
9244 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
9245 case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op1);
9246 case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op1);
9247 case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op1);
9248 case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op1);
9249 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
9250 case MVT::v32i8: return fastEmit_ISD_AND_MVT_v32i8_rr(RetVT, Op0, Op1);
9251 case MVT::v64i8: return fastEmit_ISD_AND_MVT_v64i8_rr(RetVT, Op0, Op1);
9252 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
9253 case MVT::v16i16: return fastEmit_ISD_AND_MVT_v16i16_rr(RetVT, Op0, Op1);
9254 case MVT::v32i16: return fastEmit_ISD_AND_MVT_v32i16_rr(RetVT, Op0, Op1);
9255 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
9256 case MVT::v8i32: return fastEmit_ISD_AND_MVT_v8i32_rr(RetVT, Op0, Op1);
9257 case MVT::v16i32: return fastEmit_ISD_AND_MVT_v16i32_rr(RetVT, Op0, Op1);
9258 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
9259 case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op1);
9260 case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op1);
9261 default: return 0;
9262 }
9263}
9264
9265// FastEmit functions for ISD::AVGCEILU.
9266
9267unsigned fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9268 if (RetVT.SimpleTy != MVT::v16i8)
9269 return 0;
9270 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9271 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9272 }
9273 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9274 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
9275 }
9276 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9277 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
9278 }
9279 return 0;
9280}
9281
9282unsigned fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9283 if (RetVT.SimpleTy != MVT::v32i8)
9284 return 0;
9285 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9286 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9287 }
9288 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9289 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBYrr, RC: &X86::VR256RegClass, Op0, Op1);
9290 }
9291 return 0;
9292}
9293
9294unsigned fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9295 if (RetVT.SimpleTy != MVT::v64i8)
9296 return 0;
9297 if ((Subtarget->hasBWI())) {
9298 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZrr, RC: &X86::VR512RegClass, Op0, Op1);
9299 }
9300 return 0;
9301}
9302
9303unsigned fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9304 if (RetVT.SimpleTy != MVT::v8i16)
9305 return 0;
9306 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9307 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9308 }
9309 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9310 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
9311 }
9312 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9313 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
9314 }
9315 return 0;
9316}
9317
9318unsigned fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9319 if (RetVT.SimpleTy != MVT::v16i16)
9320 return 0;
9321 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9322 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9323 }
9324 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9325 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWYrr, RC: &X86::VR256RegClass, Op0, Op1);
9326 }
9327 return 0;
9328}
9329
9330unsigned fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9331 if (RetVT.SimpleTy != MVT::v32i16)
9332 return 0;
9333 if ((Subtarget->hasBWI())) {
9334 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZrr, RC: &X86::VR512RegClass, Op0, Op1);
9335 }
9336 return 0;
9337}
9338
9339unsigned fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9340 switch (VT.SimpleTy) {
9341 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
9342 case MVT::v32i8: return fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(RetVT, Op0, Op1);
9343 case MVT::v64i8: return fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(RetVT, Op0, Op1);
9344 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
9345 case MVT::v16i16: return fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(RetVT, Op0, Op1);
9346 case MVT::v32i16: return fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(RetVT, Op0, Op1);
9347 default: return 0;
9348 }
9349}
9350
9351// FastEmit functions for ISD::FADD.
9352
9353unsigned fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9354 if (RetVT.SimpleTy != MVT::f16)
9355 return 0;
9356 if ((Subtarget->hasFP16())) {
9357 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
9358 }
9359 return 0;
9360}
9361
9362unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9363 if (RetVT.SimpleTy != MVT::f32)
9364 return 0;
9365 if ((Subtarget->hasAVX512())) {
9366 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
9367 }
9368 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9369 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
9370 }
9371 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9372 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
9373 }
9374 if ((!Subtarget->hasSSE1())) {
9375 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
9376 }
9377 return 0;
9378}
9379
9380unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9381 if (RetVT.SimpleTy != MVT::f64)
9382 return 0;
9383 if ((Subtarget->hasAVX512())) {
9384 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
9385 }
9386 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9387 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
9388 }
9389 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9390 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
9391 }
9392 if ((!Subtarget->hasSSE2())) {
9393 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
9394 }
9395 return 0;
9396}
9397
9398unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9399 if (RetVT.SimpleTy != MVT::f80)
9400 return 0;
9401 if ((Subtarget->hasX87())) {
9402 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
9403 }
9404 return 0;
9405}
9406
9407unsigned fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9408 if (RetVT.SimpleTy != MVT::v8f16)
9409 return 0;
9410 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9411 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9412 }
9413 return 0;
9414}
9415
9416unsigned fastEmit_ISD_FADD_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9417 if (RetVT.SimpleTy != MVT::v16f16)
9418 return 0;
9419 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9420 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9421 }
9422 return 0;
9423}
9424
9425unsigned fastEmit_ISD_FADD_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9426 if (RetVT.SimpleTy != MVT::v32f16)
9427 return 0;
9428 if ((Subtarget->hasFP16())) {
9429 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
9430 }
9431 return 0;
9432}
9433
9434unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9435 if (RetVT.SimpleTy != MVT::v4f32)
9436 return 0;
9437 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9438 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9439 }
9440 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9441 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
9442 }
9443 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9444 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
9445 }
9446 return 0;
9447}
9448
9449unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9450 if (RetVT.SimpleTy != MVT::v8f32)
9451 return 0;
9452 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9453 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9454 }
9455 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9456 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
9457 }
9458 return 0;
9459}
9460
9461unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9462 if (RetVT.SimpleTy != MVT::v16f32)
9463 return 0;
9464 if ((Subtarget->hasAVX512())) {
9465 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
9466 }
9467 return 0;
9468}
9469
9470unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9471 if (RetVT.SimpleTy != MVT::v2f64)
9472 return 0;
9473 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9474 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9475 }
9476 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9477 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
9478 }
9479 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9480 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
9481 }
9482 return 0;
9483}
9484
9485unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9486 if (RetVT.SimpleTy != MVT::v4f64)
9487 return 0;
9488 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9489 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9490 }
9491 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9492 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
9493 }
9494 return 0;
9495}
9496
9497unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9498 if (RetVT.SimpleTy != MVT::v8f64)
9499 return 0;
9500 if ((Subtarget->hasAVX512())) {
9501 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
9502 }
9503 return 0;
9504}
9505
9506unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9507 switch (VT.SimpleTy) {
9508 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
9509 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
9510 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
9511 case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op1);
9512 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
9513 case MVT::v16f16: return fastEmit_ISD_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
9514 case MVT::v32f16: return fastEmit_ISD_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
9515 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
9516 case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
9517 case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
9518 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
9519 case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
9520 case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
9521 default: return 0;
9522 }
9523}
9524
9525// FastEmit functions for ISD::FDIV.
9526
9527unsigned fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9528 if (RetVT.SimpleTy != MVT::f16)
9529 return 0;
9530 if ((Subtarget->hasFP16())) {
9531 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
9532 }
9533 return 0;
9534}
9535
9536unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9537 if (RetVT.SimpleTy != MVT::f32)
9538 return 0;
9539 if ((Subtarget->hasAVX512())) {
9540 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
9541 }
9542 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9543 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
9544 }
9545 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9546 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
9547 }
9548 if ((!Subtarget->hasSSE1())) {
9549 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
9550 }
9551 return 0;
9552}
9553
9554unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9555 if (RetVT.SimpleTy != MVT::f64)
9556 return 0;
9557 if ((Subtarget->hasAVX512())) {
9558 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
9559 }
9560 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9561 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
9562 }
9563 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9564 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
9565 }
9566 if ((!Subtarget->hasSSE2())) {
9567 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
9568 }
9569 return 0;
9570}
9571
9572unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9573 if (RetVT.SimpleTy != MVT::f80)
9574 return 0;
9575 if ((Subtarget->hasX87())) {
9576 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
9577 }
9578 return 0;
9579}
9580
9581unsigned fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9582 if (RetVT.SimpleTy != MVT::v8f16)
9583 return 0;
9584 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9585 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9586 }
9587 return 0;
9588}
9589
9590unsigned fastEmit_ISD_FDIV_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9591 if (RetVT.SimpleTy != MVT::v16f16)
9592 return 0;
9593 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9594 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9595 }
9596 return 0;
9597}
9598
9599unsigned fastEmit_ISD_FDIV_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9600 if (RetVT.SimpleTy != MVT::v32f16)
9601 return 0;
9602 if ((Subtarget->hasFP16())) {
9603 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
9604 }
9605 return 0;
9606}
9607
9608unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9609 if (RetVT.SimpleTy != MVT::v4f32)
9610 return 0;
9611 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9612 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9613 }
9614 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9615 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
9616 }
9617 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9618 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
9619 }
9620 return 0;
9621}
9622
9623unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9624 if (RetVT.SimpleTy != MVT::v8f32)
9625 return 0;
9626 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9627 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9628 }
9629 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9630 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
9631 }
9632 return 0;
9633}
9634
9635unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9636 if (RetVT.SimpleTy != MVT::v16f32)
9637 return 0;
9638 if ((Subtarget->hasAVX512())) {
9639 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
9640 }
9641 return 0;
9642}
9643
9644unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9645 if (RetVT.SimpleTy != MVT::v2f64)
9646 return 0;
9647 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9648 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9649 }
9650 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9651 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
9652 }
9653 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9654 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
9655 }
9656 return 0;
9657}
9658
9659unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9660 if (RetVT.SimpleTy != MVT::v4f64)
9661 return 0;
9662 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9663 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9664 }
9665 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9666 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
9667 }
9668 return 0;
9669}
9670
9671unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9672 if (RetVT.SimpleTy != MVT::v8f64)
9673 return 0;
9674 if ((Subtarget->hasAVX512())) {
9675 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
9676 }
9677 return 0;
9678}
9679
9680unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9681 switch (VT.SimpleTy) {
9682 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
9683 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
9684 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
9685 case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
9686 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
9687 case MVT::v16f16: return fastEmit_ISD_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
9688 case MVT::v32f16: return fastEmit_ISD_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
9689 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
9690 case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
9691 case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
9692 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
9693 case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
9694 case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
9695 default: return 0;
9696 }
9697}
9698
9699// FastEmit functions for ISD::FMUL.
9700
9701unsigned fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9702 if (RetVT.SimpleTy != MVT::f16)
9703 return 0;
9704 if ((Subtarget->hasFP16())) {
9705 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
9706 }
9707 return 0;
9708}
9709
9710unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9711 if (RetVT.SimpleTy != MVT::f32)
9712 return 0;
9713 if ((Subtarget->hasAVX512())) {
9714 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
9715 }
9716 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9717 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
9718 }
9719 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9720 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
9721 }
9722 if ((!Subtarget->hasSSE1())) {
9723 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
9724 }
9725 return 0;
9726}
9727
9728unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9729 if (RetVT.SimpleTy != MVT::f64)
9730 return 0;
9731 if ((Subtarget->hasAVX512())) {
9732 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
9733 }
9734 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9735 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
9736 }
9737 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9738 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
9739 }
9740 if ((!Subtarget->hasSSE2())) {
9741 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
9742 }
9743 return 0;
9744}
9745
9746unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9747 if (RetVT.SimpleTy != MVT::f80)
9748 return 0;
9749 if ((Subtarget->hasX87())) {
9750 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
9751 }
9752 return 0;
9753}
9754
9755unsigned fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9756 if (RetVT.SimpleTy != MVT::v8f16)
9757 return 0;
9758 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9759 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9760 }
9761 return 0;
9762}
9763
9764unsigned fastEmit_ISD_FMUL_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9765 if (RetVT.SimpleTy != MVT::v16f16)
9766 return 0;
9767 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9768 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9769 }
9770 return 0;
9771}
9772
9773unsigned fastEmit_ISD_FMUL_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9774 if (RetVT.SimpleTy != MVT::v32f16)
9775 return 0;
9776 if ((Subtarget->hasFP16())) {
9777 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
9778 }
9779 return 0;
9780}
9781
9782unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9783 if (RetVT.SimpleTy != MVT::v4f32)
9784 return 0;
9785 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9786 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9787 }
9788 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9789 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
9790 }
9791 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9792 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
9793 }
9794 return 0;
9795}
9796
9797unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9798 if (RetVT.SimpleTy != MVT::v8f32)
9799 return 0;
9800 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9801 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9802 }
9803 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9804 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
9805 }
9806 return 0;
9807}
9808
9809unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9810 if (RetVT.SimpleTy != MVT::v16f32)
9811 return 0;
9812 if ((Subtarget->hasAVX512())) {
9813 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
9814 }
9815 return 0;
9816}
9817
9818unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9819 if (RetVT.SimpleTy != MVT::v2f64)
9820 return 0;
9821 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9822 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9823 }
9824 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9825 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
9826 }
9827 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9828 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
9829 }
9830 return 0;
9831}
9832
9833unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9834 if (RetVT.SimpleTy != MVT::v4f64)
9835 return 0;
9836 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9837 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9838 }
9839 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9840 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
9841 }
9842 return 0;
9843}
9844
9845unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9846 if (RetVT.SimpleTy != MVT::v8f64)
9847 return 0;
9848 if ((Subtarget->hasAVX512())) {
9849 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
9850 }
9851 return 0;
9852}
9853
9854unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9855 switch (VT.SimpleTy) {
9856 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
9857 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
9858 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
9859 case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
9860 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
9861 case MVT::v16f16: return fastEmit_ISD_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
9862 case MVT::v32f16: return fastEmit_ISD_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
9863 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
9864 case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
9865 case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
9866 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
9867 case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
9868 case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
9869 default: return 0;
9870 }
9871}
9872
9873// FastEmit functions for ISD::FSUB.
9874
9875unsigned fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9876 if (RetVT.SimpleTy != MVT::f16)
9877 return 0;
9878 if ((Subtarget->hasFP16())) {
9879 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
9880 }
9881 return 0;
9882}
9883
9884unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9885 if (RetVT.SimpleTy != MVT::f32)
9886 return 0;
9887 if ((Subtarget->hasAVX512())) {
9888 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
9889 }
9890 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9891 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
9892 }
9893 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9894 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
9895 }
9896 if ((!Subtarget->hasSSE1())) {
9897 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
9898 }
9899 return 0;
9900}
9901
9902unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9903 if (RetVT.SimpleTy != MVT::f64)
9904 return 0;
9905 if ((Subtarget->hasAVX512())) {
9906 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
9907 }
9908 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9909 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
9910 }
9911 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9912 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
9913 }
9914 if ((!Subtarget->hasSSE2())) {
9915 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
9916 }
9917 return 0;
9918}
9919
9920unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9921 if (RetVT.SimpleTy != MVT::f80)
9922 return 0;
9923 if ((Subtarget->hasX87())) {
9924 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
9925 }
9926 return 0;
9927}
9928
9929unsigned fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9930 if (RetVT.SimpleTy != MVT::v8f16)
9931 return 0;
9932 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9933 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9934 }
9935 return 0;
9936}
9937
9938unsigned fastEmit_ISD_FSUB_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9939 if (RetVT.SimpleTy != MVT::v16f16)
9940 return 0;
9941 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9942 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9943 }
9944 return 0;
9945}
9946
9947unsigned fastEmit_ISD_FSUB_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9948 if (RetVT.SimpleTy != MVT::v32f16)
9949 return 0;
9950 if ((Subtarget->hasFP16())) {
9951 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
9952 }
9953 return 0;
9954}
9955
9956unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9957 if (RetVT.SimpleTy != MVT::v4f32)
9958 return 0;
9959 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9960 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9961 }
9962 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9963 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
9964 }
9965 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9966 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
9967 }
9968 return 0;
9969}
9970
9971unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9972 if (RetVT.SimpleTy != MVT::v8f32)
9973 return 0;
9974 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9975 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
9976 }
9977 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9978 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
9979 }
9980 return 0;
9981}
9982
9983unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9984 if (RetVT.SimpleTy != MVT::v16f32)
9985 return 0;
9986 if ((Subtarget->hasAVX512())) {
9987 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
9988 }
9989 return 0;
9990}
9991
9992unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9993 if (RetVT.SimpleTy != MVT::v2f64)
9994 return 0;
9995 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9996 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
9997 }
9998 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9999 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
10000 }
10001 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10002 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
10003 }
10004 return 0;
10005}
10006
10007unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10008 if (RetVT.SimpleTy != MVT::v4f64)
10009 return 0;
10010 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10011 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10012 }
10013 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10014 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10015 }
10016 return 0;
10017}
10018
10019unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10020 if (RetVT.SimpleTy != MVT::v8f64)
10021 return 0;
10022 if ((Subtarget->hasAVX512())) {
10023 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10024 }
10025 return 0;
10026}
10027
10028unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10029 switch (VT.SimpleTy) {
10030 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
10031 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
10032 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
10033 case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
10034 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
10035 case MVT::v16f16: return fastEmit_ISD_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
10036 case MVT::v32f16: return fastEmit_ISD_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
10037 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
10038 case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
10039 case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
10040 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
10041 case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
10042 case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
10043 default: return 0;
10044 }
10045}
10046
10047// FastEmit functions for ISD::MUL.
10048
10049unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10050 if (RetVT.SimpleTy != MVT::i8)
10051 return 0;
10052 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::AL).addReg(RegNo: Op0);
10053 return fastEmitInst_r(MachineInstOpcode: X86::MUL8r, RC: &X86::GR8RegClass, Op0: Op1);
10054}
10055
10056unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10057 if (RetVT.SimpleTy != MVT::i16)
10058 return 0;
10059 if ((Subtarget->hasNDD())) {
10060 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10061 }
10062 if ((!Subtarget->hasNDD())) {
10063 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr, RC: &X86::GR16RegClass, Op0, Op1);
10064 }
10065 return 0;
10066}
10067
10068unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10069 if (RetVT.SimpleTy != MVT::i32)
10070 return 0;
10071 if ((Subtarget->hasNDD())) {
10072 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10073 }
10074 if ((!Subtarget->hasNDD())) {
10075 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr, RC: &X86::GR32RegClass, Op0, Op1);
10076 }
10077 return 0;
10078}
10079
10080unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10081 if (RetVT.SimpleTy != MVT::i64)
10082 return 0;
10083 if ((Subtarget->hasNDD())) {
10084 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10085 }
10086 if ((!Subtarget->hasNDD())) {
10087 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr, RC: &X86::GR64RegClass, Op0, Op1);
10088 }
10089 return 0;
10090}
10091
10092unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10093 if (RetVT.SimpleTy != MVT::v8i16)
10094 return 0;
10095 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10096 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10097 }
10098 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10099 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
10100 }
10101 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10102 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
10103 }
10104 return 0;
10105}
10106
10107unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10108 if (RetVT.SimpleTy != MVT::v16i16)
10109 return 0;
10110 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10111 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10112 }
10113 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10114 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10115 }
10116 return 0;
10117}
10118
10119unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10120 if (RetVT.SimpleTy != MVT::v32i16)
10121 return 0;
10122 if ((Subtarget->hasBWI())) {
10123 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10124 }
10125 return 0;
10126}
10127
10128unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10129 if (RetVT.SimpleTy != MVT::v4i32)
10130 return 0;
10131 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10132 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10133 }
10134 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10135 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
10136 }
10137 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10138 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
10139 }
10140 return 0;
10141}
10142
10143unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10144 if (RetVT.SimpleTy != MVT::v8i32)
10145 return 0;
10146 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10147 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10148 }
10149 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10150 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10151 }
10152 return 0;
10153}
10154
10155unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10156 if (RetVT.SimpleTy != MVT::v16i32)
10157 return 0;
10158 if ((Subtarget->hasAVX512())) {
10159 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10160 }
10161 return 0;
10162}
10163
10164unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10165 if (RetVT.SimpleTy != MVT::v2i64)
10166 return 0;
10167 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
10168 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10169 }
10170 return 0;
10171}
10172
10173unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10174 if (RetVT.SimpleTy != MVT::v4i64)
10175 return 0;
10176 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
10177 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10178 }
10179 return 0;
10180}
10181
10182unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10183 if (RetVT.SimpleTy != MVT::v8i64)
10184 return 0;
10185 if ((Subtarget->hasDQI())) {
10186 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10187 }
10188 return 0;
10189}
10190
10191unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10192 switch (VT.SimpleTy) {
10193 case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op1);
10194 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op1);
10195 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
10196 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op1);
10197 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
10198 case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op1);
10199 case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op1);
10200 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
10201 case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op1);
10202 case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op1);
10203 case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op1);
10204 case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op1);
10205 case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op1);
10206 default: return 0;
10207 }
10208}
10209
10210// FastEmit functions for ISD::MULHS.
10211
10212unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10213 if (RetVT.SimpleTy != MVT::v8i16)
10214 return 0;
10215 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10216 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10217 }
10218 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10219 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
10220 }
10221 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10222 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
10223 }
10224 return 0;
10225}
10226
10227unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10228 if (RetVT.SimpleTy != MVT::v16i16)
10229 return 0;
10230 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10231 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10232 }
10233 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10234 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10235 }
10236 return 0;
10237}
10238
10239unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10240 if (RetVT.SimpleTy != MVT::v32i16)
10241 return 0;
10242 if ((Subtarget->hasBWI())) {
10243 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10244 }
10245 return 0;
10246}
10247
10248unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10249 switch (VT.SimpleTy) {
10250 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
10251 case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op1);
10252 case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op1);
10253 default: return 0;
10254 }
10255}
10256
10257// FastEmit functions for ISD::MULHU.
10258
10259unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10260 if (RetVT.SimpleTy != MVT::v8i16)
10261 return 0;
10262 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10263 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10264 }
10265 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10266 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
10267 }
10268 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10269 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
10270 }
10271 return 0;
10272}
10273
10274unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10275 if (RetVT.SimpleTy != MVT::v16i16)
10276 return 0;
10277 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10278 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10279 }
10280 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10281 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10282 }
10283 return 0;
10284}
10285
10286unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10287 if (RetVT.SimpleTy != MVT::v32i16)
10288 return 0;
10289 if ((Subtarget->hasBWI())) {
10290 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10291 }
10292 return 0;
10293}
10294
10295unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10296 switch (VT.SimpleTy) {
10297 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
10298 case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op1);
10299 case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op1);
10300 default: return 0;
10301 }
10302}
10303
10304// FastEmit functions for ISD::OR.
10305
10306unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10307 if (RetVT.SimpleTy != MVT::i8)
10308 return 0;
10309 if ((Subtarget->hasNDD())) {
10310 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10311 }
10312 if ((!Subtarget->hasNDD())) {
10313 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr, RC: &X86::GR8RegClass, Op0, Op1);
10314 }
10315 return 0;
10316}
10317
10318unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10319 if (RetVT.SimpleTy != MVT::i16)
10320 return 0;
10321 if ((Subtarget->hasNDD())) {
10322 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10323 }
10324 if ((!Subtarget->hasNDD())) {
10325 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr, RC: &X86::GR16RegClass, Op0, Op1);
10326 }
10327 return 0;
10328}
10329
10330unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10331 if (RetVT.SimpleTy != MVT::i32)
10332 return 0;
10333 if ((Subtarget->hasNDD())) {
10334 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10335 }
10336 if ((!Subtarget->hasNDD())) {
10337 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr, RC: &X86::GR32RegClass, Op0, Op1);
10338 }
10339 return 0;
10340}
10341
10342unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10343 if (RetVT.SimpleTy != MVT::i64)
10344 return 0;
10345 if ((Subtarget->hasNDD())) {
10346 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10347 }
10348 if ((!Subtarget->hasNDD())) {
10349 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr, RC: &X86::GR64RegClass, Op0, Op1);
10350 }
10351 return 0;
10352}
10353
10354unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10355 if (RetVT.SimpleTy != MVT::v8i1)
10356 return 0;
10357 if ((Subtarget->hasDQI())) {
10358 return fastEmitInst_rr(MachineInstOpcode: X86::KORBrr, RC: &X86::VK8RegClass, Op0, Op1);
10359 }
10360 return 0;
10361}
10362
10363unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10364 if (RetVT.SimpleTy != MVT::v16i1)
10365 return 0;
10366 if ((Subtarget->hasAVX512())) {
10367 return fastEmitInst_rr(MachineInstOpcode: X86::KORWrr, RC: &X86::VK16RegClass, Op0, Op1);
10368 }
10369 return 0;
10370}
10371
10372unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10373 if (RetVT.SimpleTy != MVT::v32i1)
10374 return 0;
10375 if ((Subtarget->hasBWI())) {
10376 return fastEmitInst_rr(MachineInstOpcode: X86::KORDrr, RC: &X86::VK32RegClass, Op0, Op1);
10377 }
10378 return 0;
10379}
10380
10381unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10382 if (RetVT.SimpleTy != MVT::v64i1)
10383 return 0;
10384 if ((Subtarget->hasBWI())) {
10385 return fastEmitInst_rr(MachineInstOpcode: X86::KORQrr, RC: &X86::VK64RegClass, Op0, Op1);
10386 }
10387 return 0;
10388}
10389
10390unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10391 if (RetVT.SimpleTy != MVT::v16i8)
10392 return 0;
10393 if ((Subtarget->hasVLX())) {
10394 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10395 }
10396 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10397 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
10398 }
10399 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10400 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
10401 }
10402 return 0;
10403}
10404
10405unsigned fastEmit_ISD_OR_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10406 if (RetVT.SimpleTy != MVT::v32i8)
10407 return 0;
10408 if ((Subtarget->hasVLX())) {
10409 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10410 }
10411 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10412 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10413 }
10414 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10415 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
10416 }
10417 return 0;
10418}
10419
10420unsigned fastEmit_ISD_OR_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10421 if (RetVT.SimpleTy != MVT::v64i8)
10422 return 0;
10423 if ((Subtarget->hasAVX512())) {
10424 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10425 }
10426 return 0;
10427}
10428
10429unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10430 if (RetVT.SimpleTy != MVT::v8i16)
10431 return 0;
10432 if ((Subtarget->hasVLX())) {
10433 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10434 }
10435 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10436 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
10437 }
10438 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10439 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
10440 }
10441 return 0;
10442}
10443
10444unsigned fastEmit_ISD_OR_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10445 if (RetVT.SimpleTy != MVT::v16i16)
10446 return 0;
10447 if ((Subtarget->hasVLX())) {
10448 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10449 }
10450 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10451 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10452 }
10453 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10454 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
10455 }
10456 return 0;
10457}
10458
10459unsigned fastEmit_ISD_OR_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10460 if (RetVT.SimpleTy != MVT::v32i16)
10461 return 0;
10462 if ((Subtarget->hasAVX512())) {
10463 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10464 }
10465 return 0;
10466}
10467
10468unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10469 if (RetVT.SimpleTy != MVT::v4i32)
10470 return 0;
10471 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10472 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
10473 }
10474 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10475 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
10476 }
10477 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10478 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10479 }
10480 return 0;
10481}
10482
10483unsigned fastEmit_ISD_OR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10484 if (RetVT.SimpleTy != MVT::v8i32)
10485 return 0;
10486 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10487 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10488 }
10489 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10490 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
10491 }
10492 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10493 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10494 }
10495 return 0;
10496}
10497
10498unsigned fastEmit_ISD_OR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10499 if (RetVT.SimpleTy != MVT::v16i32)
10500 return 0;
10501 if ((Subtarget->hasAVX512())) {
10502 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10503 }
10504 return 0;
10505}
10506
10507unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10508 if (RetVT.SimpleTy != MVT::v2i64)
10509 return 0;
10510 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10511 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10512 }
10513 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10514 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
10515 }
10516 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10517 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
10518 }
10519 return 0;
10520}
10521
10522unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10523 if (RetVT.SimpleTy != MVT::v4i64)
10524 return 0;
10525 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10526 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10527 }
10528 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10529 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10530 }
10531 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10532 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
10533 }
10534 return 0;
10535}
10536
10537unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10538 if (RetVT.SimpleTy != MVT::v8i64)
10539 return 0;
10540 if ((Subtarget->hasAVX512())) {
10541 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10542 }
10543 return 0;
10544}
10545
10546unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10547 switch (VT.SimpleTy) {
10548 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op1);
10549 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op1);
10550 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
10551 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
10552 case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op1);
10553 case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op1);
10554 case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op1);
10555 case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op1);
10556 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
10557 case MVT::v32i8: return fastEmit_ISD_OR_MVT_v32i8_rr(RetVT, Op0, Op1);
10558 case MVT::v64i8: return fastEmit_ISD_OR_MVT_v64i8_rr(RetVT, Op0, Op1);
10559 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
10560 case MVT::v16i16: return fastEmit_ISD_OR_MVT_v16i16_rr(RetVT, Op0, Op1);
10561 case MVT::v32i16: return fastEmit_ISD_OR_MVT_v32i16_rr(RetVT, Op0, Op1);
10562 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
10563 case MVT::v8i32: return fastEmit_ISD_OR_MVT_v8i32_rr(RetVT, Op0, Op1);
10564 case MVT::v16i32: return fastEmit_ISD_OR_MVT_v16i32_rr(RetVT, Op0, Op1);
10565 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
10566 case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op1);
10567 case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op1);
10568 default: return 0;
10569 }
10570}
10571
10572// FastEmit functions for ISD::ROTL.
10573
10574unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10575 if (RetVT.SimpleTy != MVT::i8)
10576 return 0;
10577 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
10578 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
10579 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL_ND, RC: &X86::GR8RegClass, Op0);
10580 }
10581 if ((!Subtarget->hasNDD())) {
10582 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
10583 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL, RC: &X86::GR8RegClass, Op0);
10584 }
10585 return 0;
10586}
10587
10588unsigned fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10589 if (RetVT.SimpleTy != MVT::v16i8)
10590 return 0;
10591 if ((Subtarget->hasXOP())) {
10592 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTBrr, RC: &X86::VR128RegClass, Op0, Op1);
10593 }
10594 return 0;
10595}
10596
10597unsigned fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10598 if (RetVT.SimpleTy != MVT::v8i16)
10599 return 0;
10600 if ((Subtarget->hasXOP())) {
10601 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTWrr, RC: &X86::VR128RegClass, Op0, Op1);
10602 }
10603 return 0;
10604}
10605
10606unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10607 if (RetVT.SimpleTy != MVT::v4i32)
10608 return 0;
10609 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10610 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10611 }
10612 if ((Subtarget->hasXOP())) {
10613 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTDrr, RC: &X86::VR128RegClass, Op0, Op1);
10614 }
10615 return 0;
10616}
10617
10618unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10619 if (RetVT.SimpleTy != MVT::v8i32)
10620 return 0;
10621 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10622 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10623 }
10624 return 0;
10625}
10626
10627unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10628 if (RetVT.SimpleTy != MVT::v16i32)
10629 return 0;
10630 if ((Subtarget->hasAVX512())) {
10631 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10632 }
10633 return 0;
10634}
10635
10636unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10637 if (RetVT.SimpleTy != MVT::v2i64)
10638 return 0;
10639 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10640 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10641 }
10642 if ((Subtarget->hasXOP())) {
10643 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTQrr, RC: &X86::VR128RegClass, Op0, Op1);
10644 }
10645 return 0;
10646}
10647
10648unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10649 if (RetVT.SimpleTy != MVT::v4i64)
10650 return 0;
10651 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10652 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10653 }
10654 return 0;
10655}
10656
10657unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10658 if (RetVT.SimpleTy != MVT::v8i64)
10659 return 0;
10660 if ((Subtarget->hasAVX512())) {
10661 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10662 }
10663 return 0;
10664}
10665
10666unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10667 switch (VT.SimpleTy) {
10668 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op1);
10669 case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op1);
10670 case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op1);
10671 case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op1);
10672 case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op1);
10673 case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op1);
10674 case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op1);
10675 case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op1);
10676 case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op1);
10677 default: return 0;
10678 }
10679}
10680
10681// FastEmit functions for ISD::ROTR.
10682
10683unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10684 if (RetVT.SimpleTy != MVT::i8)
10685 return 0;
10686 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
10687 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
10688 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL_ND, RC: &X86::GR8RegClass, Op0);
10689 }
10690 if ((!Subtarget->hasNDD())) {
10691 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
10692 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL, RC: &X86::GR8RegClass, Op0);
10693 }
10694 return 0;
10695}
10696
10697unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10698 if (RetVT.SimpleTy != MVT::v4i32)
10699 return 0;
10700 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10701 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10702 }
10703 return 0;
10704}
10705
10706unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10707 if (RetVT.SimpleTy != MVT::v8i32)
10708 return 0;
10709 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10710 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10711 }
10712 return 0;
10713}
10714
10715unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10716 if (RetVT.SimpleTy != MVT::v16i32)
10717 return 0;
10718 if ((Subtarget->hasAVX512())) {
10719 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10720 }
10721 return 0;
10722}
10723
10724unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10725 if (RetVT.SimpleTy != MVT::v2i64)
10726 return 0;
10727 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10728 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10729 }
10730 return 0;
10731}
10732
10733unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10734 if (RetVT.SimpleTy != MVT::v4i64)
10735 return 0;
10736 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10737 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10738 }
10739 return 0;
10740}
10741
10742unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10743 if (RetVT.SimpleTy != MVT::v8i64)
10744 return 0;
10745 if ((Subtarget->hasAVX512())) {
10746 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10747 }
10748 return 0;
10749}
10750
10751unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10752 switch (VT.SimpleTy) {
10753 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op1);
10754 case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op1);
10755 case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op1);
10756 case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op1);
10757 case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op1);
10758 case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op1);
10759 case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op1);
10760 default: return 0;
10761 }
10762}
10763
10764// FastEmit functions for ISD::SADDSAT.
10765
10766unsigned fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10767 if (RetVT.SimpleTy != MVT::v16i8)
10768 return 0;
10769 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10770 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10771 }
10772 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10773 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
10774 }
10775 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10776 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
10777 }
10778 return 0;
10779}
10780
10781unsigned fastEmit_ISD_SADDSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10782 if (RetVT.SimpleTy != MVT::v32i8)
10783 return 0;
10784 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10785 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10786 }
10787 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10788 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10789 }
10790 return 0;
10791}
10792
10793unsigned fastEmit_ISD_SADDSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10794 if (RetVT.SimpleTy != MVT::v64i8)
10795 return 0;
10796 if ((Subtarget->hasBWI())) {
10797 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10798 }
10799 return 0;
10800}
10801
10802unsigned fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10803 if (RetVT.SimpleTy != MVT::v8i16)
10804 return 0;
10805 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10806 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10807 }
10808 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10809 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
10810 }
10811 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10812 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
10813 }
10814 return 0;
10815}
10816
10817unsigned fastEmit_ISD_SADDSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10818 if (RetVT.SimpleTy != MVT::v16i16)
10819 return 0;
10820 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10821 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10822 }
10823 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10824 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10825 }
10826 return 0;
10827}
10828
10829unsigned fastEmit_ISD_SADDSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10830 if (RetVT.SimpleTy != MVT::v32i16)
10831 return 0;
10832 if ((Subtarget->hasBWI())) {
10833 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10834 }
10835 return 0;
10836}
10837
10838unsigned fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10839 switch (VT.SimpleTy) {
10840 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
10841 case MVT::v32i8: return fastEmit_ISD_SADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
10842 case MVT::v64i8: return fastEmit_ISD_SADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
10843 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
10844 case MVT::v16i16: return fastEmit_ISD_SADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
10845 case MVT::v32i16: return fastEmit_ISD_SADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
10846 default: return 0;
10847 }
10848}
10849
10850// FastEmit functions for ISD::SHL.
10851
10852unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10853 if (RetVT.SimpleTy != MVT::i8)
10854 return 0;
10855 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
10856 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
10857 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL_ND, RC: &X86::GR8RegClass, Op0);
10858 }
10859 if ((!Subtarget->hasNDD())) {
10860 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
10861 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL, RC: &X86::GR8RegClass, Op0);
10862 }
10863 return 0;
10864}
10865
10866unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10867 switch (VT.SimpleTy) {
10868 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op1);
10869 default: return 0;
10870 }
10871}
10872
10873// FastEmit functions for ISD::SMAX.
10874
10875unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10876 if (RetVT.SimpleTy != MVT::v16i8)
10877 return 0;
10878 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10879 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10880 }
10881 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10882 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
10883 }
10884 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10885 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
10886 }
10887 return 0;
10888}
10889
10890unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10891 if (RetVT.SimpleTy != MVT::v32i8)
10892 return 0;
10893 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10894 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10895 }
10896 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10897 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10898 }
10899 return 0;
10900}
10901
10902unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10903 if (RetVT.SimpleTy != MVT::v64i8)
10904 return 0;
10905 if ((Subtarget->hasBWI())) {
10906 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10907 }
10908 return 0;
10909}
10910
10911unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10912 if (RetVT.SimpleTy != MVT::v8i16)
10913 return 0;
10914 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10915 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10916 }
10917 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10918 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
10919 }
10920 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10921 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
10922 }
10923 return 0;
10924}
10925
10926unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10927 if (RetVT.SimpleTy != MVT::v16i16)
10928 return 0;
10929 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10930 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10931 }
10932 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10933 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10934 }
10935 return 0;
10936}
10937
10938unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10939 if (RetVT.SimpleTy != MVT::v32i16)
10940 return 0;
10941 if ((Subtarget->hasBWI())) {
10942 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10943 }
10944 return 0;
10945}
10946
10947unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10948 if (RetVT.SimpleTy != MVT::v4i32)
10949 return 0;
10950 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10951 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10952 }
10953 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10954 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
10955 }
10956 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10957 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
10958 }
10959 return 0;
10960}
10961
10962unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10963 if (RetVT.SimpleTy != MVT::v8i32)
10964 return 0;
10965 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10966 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10967 }
10968 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10969 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10970 }
10971 return 0;
10972}
10973
10974unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10975 if (RetVT.SimpleTy != MVT::v16i32)
10976 return 0;
10977 if ((Subtarget->hasAVX512())) {
10978 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10979 }
10980 return 0;
10981}
10982
10983unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10984 if (RetVT.SimpleTy != MVT::v2i64)
10985 return 0;
10986 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10987 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10988 }
10989 return 0;
10990}
10991
10992unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10993 if (RetVT.SimpleTy != MVT::v4i64)
10994 return 0;
10995 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10996 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10997 }
10998 return 0;
10999}
11000
11001unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11002 if (RetVT.SimpleTy != MVT::v8i64)
11003 return 0;
11004 if ((Subtarget->hasAVX512())) {
11005 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
11006 }
11007 return 0;
11008}
11009
11010unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11011 switch (VT.SimpleTy) {
11012 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
11013 case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
11014 case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
11015 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
11016 case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
11017 case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
11018 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
11019 case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
11020 case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
11021 case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
11022 case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
11023 case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
11024 default: return 0;
11025 }
11026}
11027
11028// FastEmit functions for ISD::SMIN.
11029
11030unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11031 if (RetVT.SimpleTy != MVT::v16i8)
11032 return 0;
11033 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11034 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11035 }
11036 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
11037 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
11038 }
11039 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11040 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
11041 }
11042 return 0;
11043}
11044
11045unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11046 if (RetVT.SimpleTy != MVT::v32i8)
11047 return 0;
11048 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11049 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11050 }
11051 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11052 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
11053 }
11054 return 0;
11055}
11056
11057unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11058 if (RetVT.SimpleTy != MVT::v64i8)
11059 return 0;
11060 if ((Subtarget->hasBWI())) {
11061 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
11062 }
11063 return 0;
11064}
11065
11066unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11067 if (RetVT.SimpleTy != MVT::v8i16)
11068 return 0;
11069 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11070 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11071 }
11072 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11073 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
11074 }
11075 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11076 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
11077 }
11078 return 0;
11079}
11080
11081unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11082 if (RetVT.SimpleTy != MVT::v16i16)
11083 return 0;
11084 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11085 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11086 }
11087 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11088 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
11089 }
11090 return 0;
11091}
11092
11093unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11094 if (RetVT.SimpleTy != MVT::v32i16)
11095 return 0;
11096 if ((Subtarget->hasBWI())) {
11097 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11098 }
11099 return 0;
11100}
11101
11102unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11103 if (RetVT.SimpleTy != MVT::v4i32)
11104 return 0;
11105 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11106 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11107 }
11108 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
11109 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
11110 }
11111 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11112 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
11113 }
11114 return 0;
11115}
11116
11117unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11118 if (RetVT.SimpleTy != MVT::v8i32)
11119 return 0;
11120 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11121 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11122 }
11123 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11124 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11125 }
11126 return 0;
11127}
11128
11129unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11130 if (RetVT.SimpleTy != MVT::v16i32)
11131 return 0;
11132 if ((Subtarget->hasAVX512())) {
11133 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11134 }
11135 return 0;
11136}
11137
11138unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11139 if (RetVT.SimpleTy != MVT::v2i64)
11140 return 0;
11141 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11142 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11143 }
11144 return 0;
11145}
11146
11147unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11148 if (RetVT.SimpleTy != MVT::v4i64)
11149 return 0;
11150 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11151 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11152 }
11153 return 0;
11154}
11155
11156unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11157 if (RetVT.SimpleTy != MVT::v8i64)
11158 return 0;
11159 if ((Subtarget->hasAVX512())) {
11160 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
11161 }
11162 return 0;
11163}
11164
11165unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11166 switch (VT.SimpleTy) {
11167 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
11168 case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
11169 case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
11170 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
11171 case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
11172 case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
11173 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
11174 case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
11175 case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
11176 case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
11177 case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
11178 case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
11179 default: return 0;
11180 }
11181}
11182
11183// FastEmit functions for ISD::SRA.
11184
11185unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11186 if (RetVT.SimpleTy != MVT::i8)
11187 return 0;
11188 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
11189 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
11190 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL_ND, RC: &X86::GR8RegClass, Op0);
11191 }
11192 if ((!Subtarget->hasNDD())) {
11193 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
11194 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL, RC: &X86::GR8RegClass, Op0);
11195 }
11196 return 0;
11197}
11198
11199unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11200 switch (VT.SimpleTy) {
11201 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op1);
11202 default: return 0;
11203 }
11204}
11205
11206// FastEmit functions for ISD::SRL.
11207
11208unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11209 if (RetVT.SimpleTy != MVT::i8)
11210 return 0;
11211 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
11212 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
11213 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL_ND, RC: &X86::GR8RegClass, Op0);
11214 }
11215 if ((!Subtarget->hasNDD())) {
11216 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
11217 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL, RC: &X86::GR8RegClass, Op0);
11218 }
11219 return 0;
11220}
11221
11222unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11223 switch (VT.SimpleTy) {
11224 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op1);
11225 default: return 0;
11226 }
11227}
11228
11229// FastEmit functions for ISD::SSUBSAT.
11230
11231unsigned fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11232 if (RetVT.SimpleTy != MVT::v16i8)
11233 return 0;
11234 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11235 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11236 }
11237 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11238 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
11239 }
11240 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11241 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
11242 }
11243 return 0;
11244}
11245
11246unsigned fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11247 if (RetVT.SimpleTy != MVT::v32i8)
11248 return 0;
11249 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11250 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11251 }
11252 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11253 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
11254 }
11255 return 0;
11256}
11257
11258unsigned fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11259 if (RetVT.SimpleTy != MVT::v64i8)
11260 return 0;
11261 if ((Subtarget->hasBWI())) {
11262 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
11263 }
11264 return 0;
11265}
11266
11267unsigned fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11268 if (RetVT.SimpleTy != MVT::v8i16)
11269 return 0;
11270 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11271 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11272 }
11273 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11274 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
11275 }
11276 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11277 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
11278 }
11279 return 0;
11280}
11281
11282unsigned fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11283 if (RetVT.SimpleTy != MVT::v16i16)
11284 return 0;
11285 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11286 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11287 }
11288 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11289 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
11290 }
11291 return 0;
11292}
11293
11294unsigned fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11295 if (RetVT.SimpleTy != MVT::v32i16)
11296 return 0;
11297 if ((Subtarget->hasBWI())) {
11298 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11299 }
11300 return 0;
11301}
11302
11303unsigned fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11304 switch (VT.SimpleTy) {
11305 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
11306 case MVT::v32i8: return fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
11307 case MVT::v64i8: return fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
11308 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
11309 case MVT::v16i16: return fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
11310 case MVT::v32i16: return fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
11311 default: return 0;
11312 }
11313}
11314
11315// FastEmit functions for ISD::STRICT_FADD.
11316
11317unsigned fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11318 if (RetVT.SimpleTy != MVT::f16)
11319 return 0;
11320 if ((Subtarget->hasFP16())) {
11321 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11322 }
11323 return 0;
11324}
11325
11326unsigned fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11327 if (RetVT.SimpleTy != MVT::f32)
11328 return 0;
11329 if ((Subtarget->hasAVX512())) {
11330 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11331 }
11332 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11333 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11334 }
11335 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11336 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11337 }
11338 if ((!Subtarget->hasSSE1())) {
11339 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11340 }
11341 return 0;
11342}
11343
11344unsigned fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11345 if (RetVT.SimpleTy != MVT::f64)
11346 return 0;
11347 if ((Subtarget->hasAVX512())) {
11348 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11349 }
11350 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11351 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11352 }
11353 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11354 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11355 }
11356 if ((!Subtarget->hasSSE2())) {
11357 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11358 }
11359 return 0;
11360}
11361
11362unsigned fastEmit_ISD_STRICT_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11363 if (RetVT.SimpleTy != MVT::f80)
11364 return 0;
11365 if ((Subtarget->hasX87())) {
11366 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11367 }
11368 return 0;
11369}
11370
11371unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11372 if (RetVT.SimpleTy != MVT::v8f16)
11373 return 0;
11374 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11375 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11376 }
11377 return 0;
11378}
11379
11380unsigned fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11381 if (RetVT.SimpleTy != MVT::v16f16)
11382 return 0;
11383 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11384 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11385 }
11386 return 0;
11387}
11388
11389unsigned fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11390 if (RetVT.SimpleTy != MVT::v32f16)
11391 return 0;
11392 if ((Subtarget->hasFP16())) {
11393 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11394 }
11395 return 0;
11396}
11397
11398unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11399 if (RetVT.SimpleTy != MVT::v4f32)
11400 return 0;
11401 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11402 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11403 }
11404 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11405 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11406 }
11407 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11408 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11409 }
11410 return 0;
11411}
11412
11413unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11414 if (RetVT.SimpleTy != MVT::v8f32)
11415 return 0;
11416 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11417 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11418 }
11419 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11420 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11421 }
11422 return 0;
11423}
11424
11425unsigned fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11426 if (RetVT.SimpleTy != MVT::v16f32)
11427 return 0;
11428 if ((Subtarget->hasAVX512())) {
11429 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11430 }
11431 return 0;
11432}
11433
11434unsigned fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11435 if (RetVT.SimpleTy != MVT::v2f64)
11436 return 0;
11437 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11438 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11439 }
11440 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11441 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11442 }
11443 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11444 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11445 }
11446 return 0;
11447}
11448
11449unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11450 if (RetVT.SimpleTy != MVT::v4f64)
11451 return 0;
11452 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11453 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11454 }
11455 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11456 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11457 }
11458 return 0;
11459}
11460
11461unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11462 if (RetVT.SimpleTy != MVT::v8f64)
11463 return 0;
11464 if ((Subtarget->hasAVX512())) {
11465 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11466 }
11467 return 0;
11468}
11469
11470unsigned fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11471 switch (VT.SimpleTy) {
11472 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11473 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11474 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11475 case MVT::f80: return fastEmit_ISD_STRICT_FADD_MVT_f80_rr(RetVT, Op0, Op1);
11476 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11477 case MVT::v16f16: return fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
11478 case MVT::v32f16: return fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
11479 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11480 case MVT::v8f32: return fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
11481 case MVT::v16f32: return fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
11482 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11483 case MVT::v4f64: return fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
11484 case MVT::v8f64: return fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
11485 default: return 0;
11486 }
11487}
11488
11489// FastEmit functions for ISD::STRICT_FDIV.
11490
11491unsigned fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11492 if (RetVT.SimpleTy != MVT::f16)
11493 return 0;
11494 if ((Subtarget->hasFP16())) {
11495 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11496 }
11497 return 0;
11498}
11499
11500unsigned fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11501 if (RetVT.SimpleTy != MVT::f32)
11502 return 0;
11503 if ((Subtarget->hasAVX512())) {
11504 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11505 }
11506 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11507 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11508 }
11509 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11510 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11511 }
11512 if ((!Subtarget->hasSSE1())) {
11513 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11514 }
11515 return 0;
11516}
11517
11518unsigned fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11519 if (RetVT.SimpleTy != MVT::f64)
11520 return 0;
11521 if ((Subtarget->hasAVX512())) {
11522 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11523 }
11524 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11525 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11526 }
11527 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11528 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11529 }
11530 if ((!Subtarget->hasSSE2())) {
11531 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11532 }
11533 return 0;
11534}
11535
11536unsigned fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11537 if (RetVT.SimpleTy != MVT::f80)
11538 return 0;
11539 if ((Subtarget->hasX87())) {
11540 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11541 }
11542 return 0;
11543}
11544
11545unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11546 if (RetVT.SimpleTy != MVT::v8f16)
11547 return 0;
11548 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11549 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11550 }
11551 return 0;
11552}
11553
11554unsigned fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11555 if (RetVT.SimpleTy != MVT::v16f16)
11556 return 0;
11557 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11558 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11559 }
11560 return 0;
11561}
11562
11563unsigned fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11564 if (RetVT.SimpleTy != MVT::v32f16)
11565 return 0;
11566 if ((Subtarget->hasFP16())) {
11567 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11568 }
11569 return 0;
11570}
11571
11572unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11573 if (RetVT.SimpleTy != MVT::v4f32)
11574 return 0;
11575 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11576 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11577 }
11578 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11579 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11580 }
11581 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11582 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11583 }
11584 return 0;
11585}
11586
11587unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11588 if (RetVT.SimpleTy != MVT::v8f32)
11589 return 0;
11590 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11591 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11592 }
11593 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11594 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11595 }
11596 return 0;
11597}
11598
11599unsigned fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11600 if (RetVT.SimpleTy != MVT::v16f32)
11601 return 0;
11602 if ((Subtarget->hasAVX512())) {
11603 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11604 }
11605 return 0;
11606}
11607
11608unsigned fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11609 if (RetVT.SimpleTy != MVT::v2f64)
11610 return 0;
11611 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11612 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11613 }
11614 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11615 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11616 }
11617 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11618 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11619 }
11620 return 0;
11621}
11622
11623unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11624 if (RetVT.SimpleTy != MVT::v4f64)
11625 return 0;
11626 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11627 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11628 }
11629 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11630 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11631 }
11632 return 0;
11633}
11634
11635unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11636 if (RetVT.SimpleTy != MVT::v8f64)
11637 return 0;
11638 if ((Subtarget->hasAVX512())) {
11639 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11640 }
11641 return 0;
11642}
11643
11644unsigned fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11645 switch (VT.SimpleTy) {
11646 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11647 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11648 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11649 case MVT::f80: return fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
11650 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11651 case MVT::v16f16: return fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
11652 case MVT::v32f16: return fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
11653 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11654 case MVT::v8f32: return fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
11655 case MVT::v16f32: return fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
11656 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11657 case MVT::v4f64: return fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
11658 case MVT::v8f64: return fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
11659 default: return 0;
11660 }
11661}
11662
11663// FastEmit functions for ISD::STRICT_FMUL.
11664
11665unsigned fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11666 if (RetVT.SimpleTy != MVT::f16)
11667 return 0;
11668 if ((Subtarget->hasFP16())) {
11669 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11670 }
11671 return 0;
11672}
11673
11674unsigned fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11675 if (RetVT.SimpleTy != MVT::f32)
11676 return 0;
11677 if ((Subtarget->hasAVX512())) {
11678 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11679 }
11680 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11681 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11682 }
11683 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11684 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11685 }
11686 if ((!Subtarget->hasSSE1())) {
11687 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11688 }
11689 return 0;
11690}
11691
11692unsigned fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11693 if (RetVT.SimpleTy != MVT::f64)
11694 return 0;
11695 if ((Subtarget->hasAVX512())) {
11696 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11697 }
11698 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11699 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11700 }
11701 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11702 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11703 }
11704 if ((!Subtarget->hasSSE2())) {
11705 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11706 }
11707 return 0;
11708}
11709
11710unsigned fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11711 if (RetVT.SimpleTy != MVT::f80)
11712 return 0;
11713 if ((Subtarget->hasX87())) {
11714 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11715 }
11716 return 0;
11717}
11718
11719unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11720 if (RetVT.SimpleTy != MVT::v8f16)
11721 return 0;
11722 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11723 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11724 }
11725 return 0;
11726}
11727
11728unsigned fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11729 if (RetVT.SimpleTy != MVT::v16f16)
11730 return 0;
11731 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11732 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11733 }
11734 return 0;
11735}
11736
11737unsigned fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11738 if (RetVT.SimpleTy != MVT::v32f16)
11739 return 0;
11740 if ((Subtarget->hasFP16())) {
11741 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11742 }
11743 return 0;
11744}
11745
11746unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11747 if (RetVT.SimpleTy != MVT::v4f32)
11748 return 0;
11749 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11750 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11751 }
11752 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11753 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11754 }
11755 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11756 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11757 }
11758 return 0;
11759}
11760
11761unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11762 if (RetVT.SimpleTy != MVT::v8f32)
11763 return 0;
11764 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11765 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11766 }
11767 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11768 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11769 }
11770 return 0;
11771}
11772
11773unsigned fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11774 if (RetVT.SimpleTy != MVT::v16f32)
11775 return 0;
11776 if ((Subtarget->hasAVX512())) {
11777 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11778 }
11779 return 0;
11780}
11781
11782unsigned fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11783 if (RetVT.SimpleTy != MVT::v2f64)
11784 return 0;
11785 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11786 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11787 }
11788 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11789 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11790 }
11791 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11792 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11793 }
11794 return 0;
11795}
11796
11797unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11798 if (RetVT.SimpleTy != MVT::v4f64)
11799 return 0;
11800 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11801 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11802 }
11803 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11804 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11805 }
11806 return 0;
11807}
11808
11809unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11810 if (RetVT.SimpleTy != MVT::v8f64)
11811 return 0;
11812 if ((Subtarget->hasAVX512())) {
11813 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11814 }
11815 return 0;
11816}
11817
11818unsigned fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11819 switch (VT.SimpleTy) {
11820 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11821 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11822 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11823 case MVT::f80: return fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
11824 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11825 case MVT::v16f16: return fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
11826 case MVT::v32f16: return fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
11827 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11828 case MVT::v8f32: return fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
11829 case MVT::v16f32: return fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
11830 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11831 case MVT::v4f64: return fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
11832 case MVT::v8f64: return fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
11833 default: return 0;
11834 }
11835}
11836
11837// FastEmit functions for ISD::STRICT_FSUB.
11838
11839unsigned fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11840 if (RetVT.SimpleTy != MVT::f16)
11841 return 0;
11842 if ((Subtarget->hasFP16())) {
11843 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11844 }
11845 return 0;
11846}
11847
11848unsigned fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11849 if (RetVT.SimpleTy != MVT::f32)
11850 return 0;
11851 if ((Subtarget->hasAVX512())) {
11852 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11853 }
11854 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11855 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11856 }
11857 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11858 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11859 }
11860 if ((!Subtarget->hasSSE1())) {
11861 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11862 }
11863 return 0;
11864}
11865
11866unsigned fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11867 if (RetVT.SimpleTy != MVT::f64)
11868 return 0;
11869 if ((Subtarget->hasAVX512())) {
11870 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11871 }
11872 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11873 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11874 }
11875 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11876 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11877 }
11878 if ((!Subtarget->hasSSE2())) {
11879 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11880 }
11881 return 0;
11882}
11883
11884unsigned fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11885 if (RetVT.SimpleTy != MVT::f80)
11886 return 0;
11887 if ((Subtarget->hasX87())) {
11888 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11889 }
11890 return 0;
11891}
11892
11893unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11894 if (RetVT.SimpleTy != MVT::v8f16)
11895 return 0;
11896 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11897 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11898 }
11899 return 0;
11900}
11901
11902unsigned fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11903 if (RetVT.SimpleTy != MVT::v16f16)
11904 return 0;
11905 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11906 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11907 }
11908 return 0;
11909}
11910
11911unsigned fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11912 if (RetVT.SimpleTy != MVT::v32f16)
11913 return 0;
11914 if ((Subtarget->hasFP16())) {
11915 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11916 }
11917 return 0;
11918}
11919
11920unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11921 if (RetVT.SimpleTy != MVT::v4f32)
11922 return 0;
11923 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11924 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11925 }
11926 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11927 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11928 }
11929 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11930 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11931 }
11932 return 0;
11933}
11934
11935unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11936 if (RetVT.SimpleTy != MVT::v8f32)
11937 return 0;
11938 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11939 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11940 }
11941 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11942 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11943 }
11944 return 0;
11945}
11946
11947unsigned fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11948 if (RetVT.SimpleTy != MVT::v16f32)
11949 return 0;
11950 if ((Subtarget->hasAVX512())) {
11951 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11952 }
11953 return 0;
11954}
11955
11956unsigned fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11957 if (RetVT.SimpleTy != MVT::v2f64)
11958 return 0;
11959 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11960 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11961 }
11962 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11963 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11964 }
11965 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11966 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11967 }
11968 return 0;
11969}
11970
11971unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11972 if (RetVT.SimpleTy != MVT::v4f64)
11973 return 0;
11974 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11975 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11976 }
11977 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11978 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11979 }
11980 return 0;
11981}
11982
11983unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11984 if (RetVT.SimpleTy != MVT::v8f64)
11985 return 0;
11986 if ((Subtarget->hasAVX512())) {
11987 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11988 }
11989 return 0;
11990}
11991
11992unsigned fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11993 switch (VT.SimpleTy) {
11994 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11995 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11996 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11997 case MVT::f80: return fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
11998 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11999 case MVT::v16f16: return fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
12000 case MVT::v32f16: return fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
12001 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
12002 case MVT::v8f32: return fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
12003 case MVT::v16f32: return fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
12004 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
12005 case MVT::v4f64: return fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
12006 case MVT::v8f64: return fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
12007 default: return 0;
12008 }
12009}
12010
12011// FastEmit functions for ISD::SUB.
12012
12013unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12014 if (RetVT.SimpleTy != MVT::i8)
12015 return 0;
12016 if ((Subtarget->hasNDD())) {
12017 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
12018 }
12019 if ((!Subtarget->hasNDD())) {
12020 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr, RC: &X86::GR8RegClass, Op0, Op1);
12021 }
12022 return 0;
12023}
12024
12025unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12026 if (RetVT.SimpleTy != MVT::i16)
12027 return 0;
12028 if ((Subtarget->hasNDD())) {
12029 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
12030 }
12031 if ((!Subtarget->hasNDD())) {
12032 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr, RC: &X86::GR16RegClass, Op0, Op1);
12033 }
12034 return 0;
12035}
12036
12037unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12038 if (RetVT.SimpleTy != MVT::i32)
12039 return 0;
12040 if ((Subtarget->hasNDD())) {
12041 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
12042 }
12043 if ((!Subtarget->hasNDD())) {
12044 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr, RC: &X86::GR32RegClass, Op0, Op1);
12045 }
12046 return 0;
12047}
12048
12049unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12050 if (RetVT.SimpleTy != MVT::i64)
12051 return 0;
12052 if ((Subtarget->hasNDD())) {
12053 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
12054 }
12055 if ((!Subtarget->hasNDD())) {
12056 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr, RC: &X86::GR64RegClass, Op0, Op1);
12057 }
12058 return 0;
12059}
12060
12061unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12062 if (RetVT.SimpleTy != MVT::v16i8)
12063 return 0;
12064 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12065 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12066 }
12067 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12068 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
12069 }
12070 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12071 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
12072 }
12073 return 0;
12074}
12075
12076unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12077 if (RetVT.SimpleTy != MVT::v32i8)
12078 return 0;
12079 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12080 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12081 }
12082 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12083 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12084 }
12085 return 0;
12086}
12087
12088unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12089 if (RetVT.SimpleTy != MVT::v64i8)
12090 return 0;
12091 if ((Subtarget->hasBWI())) {
12092 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12093 }
12094 return 0;
12095}
12096
12097unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12098 if (RetVT.SimpleTy != MVT::v8i16)
12099 return 0;
12100 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12101 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12102 }
12103 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12104 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
12105 }
12106 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12107 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
12108 }
12109 return 0;
12110}
12111
12112unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12113 if (RetVT.SimpleTy != MVT::v16i16)
12114 return 0;
12115 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12116 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12117 }
12118 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12119 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12120 }
12121 return 0;
12122}
12123
12124unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12125 if (RetVT.SimpleTy != MVT::v32i16)
12126 return 0;
12127 if ((Subtarget->hasBWI())) {
12128 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12129 }
12130 return 0;
12131}
12132
12133unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12134 if (RetVT.SimpleTy != MVT::v4i32)
12135 return 0;
12136 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12137 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12138 }
12139 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12140 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
12141 }
12142 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12143 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
12144 }
12145 return 0;
12146}
12147
12148unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12149 if (RetVT.SimpleTy != MVT::v8i32)
12150 return 0;
12151 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12152 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12153 }
12154 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12155 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12156 }
12157 return 0;
12158}
12159
12160unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12161 if (RetVT.SimpleTy != MVT::v16i32)
12162 return 0;
12163 if ((Subtarget->hasAVX512())) {
12164 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12165 }
12166 return 0;
12167}
12168
12169unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12170 if (RetVT.SimpleTy != MVT::v2i64)
12171 return 0;
12172 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12173 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12174 }
12175 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12176 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
12177 }
12178 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12179 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
12180 }
12181 return 0;
12182}
12183
12184unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12185 if (RetVT.SimpleTy != MVT::v4i64)
12186 return 0;
12187 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12188 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12189 }
12190 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12191 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQYrr, RC: &X86::VR256RegClass, Op0, Op1);
12192 }
12193 return 0;
12194}
12195
12196unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12197 if (RetVT.SimpleTy != MVT::v8i64)
12198 return 0;
12199 if ((Subtarget->hasAVX512())) {
12200 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12201 }
12202 return 0;
12203}
12204
12205unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12206 switch (VT.SimpleTy) {
12207 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op1);
12208 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op1);
12209 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
12210 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
12211 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
12212 case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op1);
12213 case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op1);
12214 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
12215 case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op1);
12216 case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op1);
12217 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
12218 case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op1);
12219 case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op1);
12220 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
12221 case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op1);
12222 case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op1);
12223 default: return 0;
12224 }
12225}
12226
12227// FastEmit functions for ISD::UADDSAT.
12228
12229unsigned fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12230 if (RetVT.SimpleTy != MVT::v16i8)
12231 return 0;
12232 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12233 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12234 }
12235 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12236 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12237 }
12238 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12239 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12240 }
12241 return 0;
12242}
12243
12244unsigned fastEmit_ISD_UADDSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12245 if (RetVT.SimpleTy != MVT::v32i8)
12246 return 0;
12247 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12248 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12249 }
12250 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12251 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12252 }
12253 return 0;
12254}
12255
12256unsigned fastEmit_ISD_UADDSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12257 if (RetVT.SimpleTy != MVT::v64i8)
12258 return 0;
12259 if ((Subtarget->hasBWI())) {
12260 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12261 }
12262 return 0;
12263}
12264
12265unsigned fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12266 if (RetVT.SimpleTy != MVT::v8i16)
12267 return 0;
12268 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12269 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12270 }
12271 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12272 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12273 }
12274 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12275 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12276 }
12277 return 0;
12278}
12279
12280unsigned fastEmit_ISD_UADDSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12281 if (RetVT.SimpleTy != MVT::v16i16)
12282 return 0;
12283 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12284 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12285 }
12286 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12287 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12288 }
12289 return 0;
12290}
12291
12292unsigned fastEmit_ISD_UADDSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12293 if (RetVT.SimpleTy != MVT::v32i16)
12294 return 0;
12295 if ((Subtarget->hasBWI())) {
12296 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12297 }
12298 return 0;
12299}
12300
12301unsigned fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12302 switch (VT.SimpleTy) {
12303 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12304 case MVT::v32i8: return fastEmit_ISD_UADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
12305 case MVT::v64i8: return fastEmit_ISD_UADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
12306 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12307 case MVT::v16i16: return fastEmit_ISD_UADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
12308 case MVT::v32i16: return fastEmit_ISD_UADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
12309 default: return 0;
12310 }
12311}
12312
12313// FastEmit functions for ISD::UMAX.
12314
12315unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12316 if (RetVT.SimpleTy != MVT::v16i8)
12317 return 0;
12318 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12319 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12320 }
12321 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12322 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
12323 }
12324 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12325 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
12326 }
12327 return 0;
12328}
12329
12330unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12331 if (RetVT.SimpleTy != MVT::v32i8)
12332 return 0;
12333 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12334 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12335 }
12336 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12337 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12338 }
12339 return 0;
12340}
12341
12342unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12343 if (RetVT.SimpleTy != MVT::v64i8)
12344 return 0;
12345 if ((Subtarget->hasBWI())) {
12346 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12347 }
12348 return 0;
12349}
12350
12351unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12352 if (RetVT.SimpleTy != MVT::v8i16)
12353 return 0;
12354 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12355 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12356 }
12357 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12358 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12359 }
12360 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12361 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12362 }
12363 return 0;
12364}
12365
12366unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12367 if (RetVT.SimpleTy != MVT::v16i16)
12368 return 0;
12369 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12370 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12371 }
12372 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12373 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12374 }
12375 return 0;
12376}
12377
12378unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12379 if (RetVT.SimpleTy != MVT::v32i16)
12380 return 0;
12381 if ((Subtarget->hasBWI())) {
12382 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12383 }
12384 return 0;
12385}
12386
12387unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12388 if (RetVT.SimpleTy != MVT::v4i32)
12389 return 0;
12390 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12391 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12392 }
12393 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12394 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
12395 }
12396 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12397 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
12398 }
12399 return 0;
12400}
12401
12402unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12403 if (RetVT.SimpleTy != MVT::v8i32)
12404 return 0;
12405 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12406 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12407 }
12408 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12409 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12410 }
12411 return 0;
12412}
12413
12414unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12415 if (RetVT.SimpleTy != MVT::v16i32)
12416 return 0;
12417 if ((Subtarget->hasAVX512())) {
12418 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12419 }
12420 return 0;
12421}
12422
12423unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12424 if (RetVT.SimpleTy != MVT::v2i64)
12425 return 0;
12426 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12427 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12428 }
12429 return 0;
12430}
12431
12432unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12433 if (RetVT.SimpleTy != MVT::v4i64)
12434 return 0;
12435 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12436 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12437 }
12438 return 0;
12439}
12440
12441unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12442 if (RetVT.SimpleTy != MVT::v8i64)
12443 return 0;
12444 if ((Subtarget->hasAVX512())) {
12445 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12446 }
12447 return 0;
12448}
12449
12450unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12451 switch (VT.SimpleTy) {
12452 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12453 case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
12454 case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
12455 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12456 case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
12457 case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
12458 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12459 case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
12460 case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
12461 case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
12462 case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
12463 case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
12464 default: return 0;
12465 }
12466}
12467
12468// FastEmit functions for ISD::UMIN.
12469
12470unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12471 if (RetVT.SimpleTy != MVT::v16i8)
12472 return 0;
12473 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12474 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12475 }
12476 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12477 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
12478 }
12479 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12480 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
12481 }
12482 return 0;
12483}
12484
12485unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12486 if (RetVT.SimpleTy != MVT::v32i8)
12487 return 0;
12488 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12489 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12490 }
12491 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12492 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12493 }
12494 return 0;
12495}
12496
12497unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12498 if (RetVT.SimpleTy != MVT::v64i8)
12499 return 0;
12500 if ((Subtarget->hasBWI())) {
12501 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12502 }
12503 return 0;
12504}
12505
12506unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12507 if (RetVT.SimpleTy != MVT::v8i16)
12508 return 0;
12509 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12510 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12511 }
12512 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12513 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12514 }
12515 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12516 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12517 }
12518 return 0;
12519}
12520
12521unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12522 if (RetVT.SimpleTy != MVT::v16i16)
12523 return 0;
12524 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12525 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12526 }
12527 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12528 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12529 }
12530 return 0;
12531}
12532
12533unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12534 if (RetVT.SimpleTy != MVT::v32i16)
12535 return 0;
12536 if ((Subtarget->hasBWI())) {
12537 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12538 }
12539 return 0;
12540}
12541
12542unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12543 if (RetVT.SimpleTy != MVT::v4i32)
12544 return 0;
12545 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12546 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12547 }
12548 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12549 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
12550 }
12551 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12552 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
12553 }
12554 return 0;
12555}
12556
12557unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12558 if (RetVT.SimpleTy != MVT::v8i32)
12559 return 0;
12560 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12561 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12562 }
12563 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12564 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12565 }
12566 return 0;
12567}
12568
12569unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12570 if (RetVT.SimpleTy != MVT::v16i32)
12571 return 0;
12572 if ((Subtarget->hasAVX512())) {
12573 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12574 }
12575 return 0;
12576}
12577
12578unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12579 if (RetVT.SimpleTy != MVT::v2i64)
12580 return 0;
12581 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12582 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12583 }
12584 return 0;
12585}
12586
12587unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12588 if (RetVT.SimpleTy != MVT::v4i64)
12589 return 0;
12590 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12591 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12592 }
12593 return 0;
12594}
12595
12596unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12597 if (RetVT.SimpleTy != MVT::v8i64)
12598 return 0;
12599 if ((Subtarget->hasAVX512())) {
12600 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12601 }
12602 return 0;
12603}
12604
12605unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12606 switch (VT.SimpleTy) {
12607 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12608 case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
12609 case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
12610 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12611 case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
12612 case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
12613 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12614 case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
12615 case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
12616 case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
12617 case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
12618 case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
12619 default: return 0;
12620 }
12621}
12622
12623// FastEmit functions for ISD::USUBSAT.
12624
12625unsigned fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12626 if (RetVT.SimpleTy != MVT::v16i8)
12627 return 0;
12628 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12629 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12630 }
12631 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12632 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12633 }
12634 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12635 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12636 }
12637 return 0;
12638}
12639
12640unsigned fastEmit_ISD_USUBSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12641 if (RetVT.SimpleTy != MVT::v32i8)
12642 return 0;
12643 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12644 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12645 }
12646 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12647 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12648 }
12649 return 0;
12650}
12651
12652unsigned fastEmit_ISD_USUBSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12653 if (RetVT.SimpleTy != MVT::v64i8)
12654 return 0;
12655 if ((Subtarget->hasBWI())) {
12656 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12657 }
12658 return 0;
12659}
12660
12661unsigned fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12662 if (RetVT.SimpleTy != MVT::v8i16)
12663 return 0;
12664 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12665 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12666 }
12667 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12668 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12669 }
12670 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12671 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12672 }
12673 return 0;
12674}
12675
12676unsigned fastEmit_ISD_USUBSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12677 if (RetVT.SimpleTy != MVT::v16i16)
12678 return 0;
12679 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12680 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12681 }
12682 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12683 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12684 }
12685 return 0;
12686}
12687
12688unsigned fastEmit_ISD_USUBSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12689 if (RetVT.SimpleTy != MVT::v32i16)
12690 return 0;
12691 if ((Subtarget->hasBWI())) {
12692 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12693 }
12694 return 0;
12695}
12696
12697unsigned fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12698 switch (VT.SimpleTy) {
12699 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12700 case MVT::v32i8: return fastEmit_ISD_USUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
12701 case MVT::v64i8: return fastEmit_ISD_USUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
12702 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12703 case MVT::v16i16: return fastEmit_ISD_USUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
12704 case MVT::v32i16: return fastEmit_ISD_USUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
12705 default: return 0;
12706 }
12707}
12708
12709// FastEmit functions for ISD::XOR.
12710
12711unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12712 if (RetVT.SimpleTy != MVT::i8)
12713 return 0;
12714 if ((Subtarget->hasNDD())) {
12715 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
12716 }
12717 if ((!Subtarget->hasNDD())) {
12718 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr, RC: &X86::GR8RegClass, Op0, Op1);
12719 }
12720 return 0;
12721}
12722
12723unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12724 if (RetVT.SimpleTy != MVT::i16)
12725 return 0;
12726 if ((Subtarget->hasNDD())) {
12727 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
12728 }
12729 if ((!Subtarget->hasNDD())) {
12730 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr, RC: &X86::GR16RegClass, Op0, Op1);
12731 }
12732 return 0;
12733}
12734
12735unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12736 if (RetVT.SimpleTy != MVT::i32)
12737 return 0;
12738 if ((Subtarget->hasNDD())) {
12739 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
12740 }
12741 if ((!Subtarget->hasNDD())) {
12742 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr, RC: &X86::GR32RegClass, Op0, Op1);
12743 }
12744 return 0;
12745}
12746
12747unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12748 if (RetVT.SimpleTy != MVT::i64)
12749 return 0;
12750 if ((Subtarget->hasNDD())) {
12751 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
12752 }
12753 if ((!Subtarget->hasNDD())) {
12754 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr, RC: &X86::GR64RegClass, Op0, Op1);
12755 }
12756 return 0;
12757}
12758
12759unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12760 if (RetVT.SimpleTy != MVT::v8i1)
12761 return 0;
12762 if ((Subtarget->hasDQI())) {
12763 return fastEmitInst_rr(MachineInstOpcode: X86::KXORBrr, RC: &X86::VK8RegClass, Op0, Op1);
12764 }
12765 return 0;
12766}
12767
12768unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12769 if (RetVT.SimpleTy != MVT::v16i1)
12770 return 0;
12771 if ((Subtarget->hasAVX512())) {
12772 return fastEmitInst_rr(MachineInstOpcode: X86::KXORWrr, RC: &X86::VK16RegClass, Op0, Op1);
12773 }
12774 return 0;
12775}
12776
12777unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12778 if (RetVT.SimpleTy != MVT::v32i1)
12779 return 0;
12780 if ((Subtarget->hasBWI())) {
12781 return fastEmitInst_rr(MachineInstOpcode: X86::KXORDrr, RC: &X86::VK32RegClass, Op0, Op1);
12782 }
12783 return 0;
12784}
12785
12786unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12787 if (RetVT.SimpleTy != MVT::v64i1)
12788 return 0;
12789 if ((Subtarget->hasBWI())) {
12790 return fastEmitInst_rr(MachineInstOpcode: X86::KXORQrr, RC: &X86::VK64RegClass, Op0, Op1);
12791 }
12792 return 0;
12793}
12794
12795unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12796 if (RetVT.SimpleTy != MVT::v16i8)
12797 return 0;
12798 if ((Subtarget->hasVLX())) {
12799 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12800 }
12801 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12802 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
12803 }
12804 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12805 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
12806 }
12807 return 0;
12808}
12809
12810unsigned fastEmit_ISD_XOR_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12811 if (RetVT.SimpleTy != MVT::v32i8)
12812 return 0;
12813 if ((Subtarget->hasVLX())) {
12814 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12815 }
12816 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12817 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12818 }
12819 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12820 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12821 }
12822 return 0;
12823}
12824
12825unsigned fastEmit_ISD_XOR_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12826 if (RetVT.SimpleTy != MVT::v64i8)
12827 return 0;
12828 if ((Subtarget->hasAVX512())) {
12829 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12830 }
12831 return 0;
12832}
12833
12834unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12835 if (RetVT.SimpleTy != MVT::v8i16)
12836 return 0;
12837 if ((Subtarget->hasVLX())) {
12838 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12839 }
12840 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12841 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
12842 }
12843 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12844 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
12845 }
12846 return 0;
12847}
12848
12849unsigned fastEmit_ISD_XOR_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12850 if (RetVT.SimpleTy != MVT::v16i16)
12851 return 0;
12852 if ((Subtarget->hasVLX())) {
12853 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12854 }
12855 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12856 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12857 }
12858 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12859 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12860 }
12861 return 0;
12862}
12863
12864unsigned fastEmit_ISD_XOR_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12865 if (RetVT.SimpleTy != MVT::v32i16)
12866 return 0;
12867 if ((Subtarget->hasAVX512())) {
12868 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12869 }
12870 return 0;
12871}
12872
12873unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12874 if (RetVT.SimpleTy != MVT::v4i32)
12875 return 0;
12876 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12877 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
12878 }
12879 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12880 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
12881 }
12882 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12883 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12884 }
12885 return 0;
12886}
12887
12888unsigned fastEmit_ISD_XOR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12889 if (RetVT.SimpleTy != MVT::v8i32)
12890 return 0;
12891 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12892 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12893 }
12894 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12895 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12896 }
12897 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12898 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12899 }
12900 return 0;
12901}
12902
12903unsigned fastEmit_ISD_XOR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12904 if (RetVT.SimpleTy != MVT::v16i32)
12905 return 0;
12906 if ((Subtarget->hasAVX512())) {
12907 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12908 }
12909 return 0;
12910}
12911
12912unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12913 if (RetVT.SimpleTy != MVT::v2i64)
12914 return 0;
12915 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12916 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12917 }
12918 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12919 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
12920 }
12921 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12922 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
12923 }
12924 return 0;
12925}
12926
12927unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12928 if (RetVT.SimpleTy != MVT::v4i64)
12929 return 0;
12930 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12931 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12932 }
12933 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12934 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12935 }
12936 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12937 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12938 }
12939 return 0;
12940}
12941
12942unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12943 if (RetVT.SimpleTy != MVT::v8i64)
12944 return 0;
12945 if ((Subtarget->hasAVX512())) {
12946 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12947 }
12948 return 0;
12949}
12950
12951unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12952 switch (VT.SimpleTy) {
12953 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op1);
12954 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op1);
12955 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
12956 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
12957 case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op1);
12958 case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op1);
12959 case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op1);
12960 case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op1);
12961 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
12962 case MVT::v32i8: return fastEmit_ISD_XOR_MVT_v32i8_rr(RetVT, Op0, Op1);
12963 case MVT::v64i8: return fastEmit_ISD_XOR_MVT_v64i8_rr(RetVT, Op0, Op1);
12964 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
12965 case MVT::v16i16: return fastEmit_ISD_XOR_MVT_v16i16_rr(RetVT, Op0, Op1);
12966 case MVT::v32i16: return fastEmit_ISD_XOR_MVT_v32i16_rr(RetVT, Op0, Op1);
12967 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
12968 case MVT::v8i32: return fastEmit_ISD_XOR_MVT_v8i32_rr(RetVT, Op0, Op1);
12969 case MVT::v16i32: return fastEmit_ISD_XOR_MVT_v16i32_rr(RetVT, Op0, Op1);
12970 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
12971 case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op1);
12972 case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op1);
12973 default: return 0;
12974 }
12975}
12976
12977// FastEmit functions for X86ISD::ADDSUB.
12978
12979unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12980 if (RetVT.SimpleTy != MVT::v4f32)
12981 return 0;
12982 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
12983 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
12984 }
12985 if ((Subtarget->hasAVX())) {
12986 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
12987 }
12988 return 0;
12989}
12990
12991unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12992 if (RetVT.SimpleTy != MVT::v8f32)
12993 return 0;
12994 if ((Subtarget->hasAVX())) {
12995 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12996 }
12997 return 0;
12998}
12999
13000unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13001 if (RetVT.SimpleTy != MVT::v2f64)
13002 return 0;
13003 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
13004 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13005 }
13006 if ((Subtarget->hasAVX())) {
13007 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13008 }
13009 return 0;
13010}
13011
13012unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13013 if (RetVT.SimpleTy != MVT::v4f64)
13014 return 0;
13015 if ((Subtarget->hasAVX())) {
13016 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13017 }
13018 return 0;
13019}
13020
13021unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13022 switch (VT.SimpleTy) {
13023 case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13024 case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
13025 case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13026 case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
13027 default: return 0;
13028 }
13029}
13030
13031// FastEmit functions for X86ISD::ANDNP.
13032
13033unsigned fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13034 if (RetVT.SimpleTy != MVT::v16i8)
13035 return 0;
13036 if ((Subtarget->hasVLX())) {
13037 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13038 }
13039 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13040 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
13041 }
13042 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13043 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
13044 }
13045 return 0;
13046}
13047
13048unsigned fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13049 if (RetVT.SimpleTy != MVT::v32i8)
13050 return 0;
13051 if ((Subtarget->hasVLX())) {
13052 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13053 }
13054 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
13055 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13056 }
13057 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13058 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
13059 }
13060 return 0;
13061}
13062
13063unsigned fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13064 if (RetVT.SimpleTy != MVT::v64i8)
13065 return 0;
13066 if ((Subtarget->hasAVX512())) {
13067 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13068 }
13069 return 0;
13070}
13071
13072unsigned fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13073 if (RetVT.SimpleTy != MVT::v8i16)
13074 return 0;
13075 if ((Subtarget->hasVLX())) {
13076 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13077 }
13078 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13079 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
13080 }
13081 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13082 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
13083 }
13084 return 0;
13085}
13086
13087unsigned fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13088 if (RetVT.SimpleTy != MVT::v16i16)
13089 return 0;
13090 if ((Subtarget->hasVLX())) {
13091 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13092 }
13093 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
13094 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13095 }
13096 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13097 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
13098 }
13099 return 0;
13100}
13101
13102unsigned fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13103 if (RetVT.SimpleTy != MVT::v32i16)
13104 return 0;
13105 if ((Subtarget->hasAVX512())) {
13106 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13107 }
13108 return 0;
13109}
13110
13111unsigned fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13112 if (RetVT.SimpleTy != MVT::v4i32)
13113 return 0;
13114 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13115 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
13116 }
13117 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13118 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
13119 }
13120 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13121 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13122 }
13123 return 0;
13124}
13125
13126unsigned fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13127 if (RetVT.SimpleTy != MVT::v8i32)
13128 return 0;
13129 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
13130 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13131 }
13132 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13133 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
13134 }
13135 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13136 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13137 }
13138 return 0;
13139}
13140
13141unsigned fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13142 if (RetVT.SimpleTy != MVT::v16i32)
13143 return 0;
13144 if ((Subtarget->hasAVX512())) {
13145 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13146 }
13147 return 0;
13148}
13149
13150unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13151 if (RetVT.SimpleTy != MVT::v2i64)
13152 return 0;
13153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13154 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13155 }
13156 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13157 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
13158 }
13159 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13160 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
13161 }
13162 return 0;
13163}
13164
13165unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13166 if (RetVT.SimpleTy != MVT::v4i64)
13167 return 0;
13168 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
13169 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13170 }
13171 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13172 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13173 }
13174 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13175 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
13176 }
13177 return 0;
13178}
13179
13180unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13181 if (RetVT.SimpleTy != MVT::v8i64)
13182 return 0;
13183 if ((Subtarget->hasAVX512())) {
13184 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13185 }
13186 return 0;
13187}
13188
13189unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13190 switch (VT.SimpleTy) {
13191 case MVT::v16i8: return fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(RetVT, Op0, Op1);
13192 case MVT::v32i8: return fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(RetVT, Op0, Op1);
13193 case MVT::v64i8: return fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(RetVT, Op0, Op1);
13194 case MVT::v8i16: return fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(RetVT, Op0, Op1);
13195 case MVT::v16i16: return fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(RetVT, Op0, Op1);
13196 case MVT::v32i16: return fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(RetVT, Op0, Op1);
13197 case MVT::v4i32: return fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(RetVT, Op0, Op1);
13198 case MVT::v8i32: return fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(RetVT, Op0, Op1);
13199 case MVT::v16i32: return fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(RetVT, Op0, Op1);
13200 case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op1);
13201 case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op1);
13202 case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op1);
13203 default: return 0;
13204 }
13205}
13206
13207// FastEmit functions for X86ISD::BEXTR.
13208
13209unsigned fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13210 if (RetVT.SimpleTy != MVT::i32)
13211 return 0;
13212 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
13213 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
13214 }
13215 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
13216 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr, RC: &X86::GR32RegClass, Op0, Op1);
13217 }
13218 return 0;
13219}
13220
13221unsigned fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13222 if (RetVT.SimpleTy != MVT::i64)
13223 return 0;
13224 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
13225 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
13226 }
13227 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
13228 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr, RC: &X86::GR64RegClass, Op0, Op1);
13229 }
13230 return 0;
13231}
13232
13233unsigned fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13234 switch (VT.SimpleTy) {
13235 case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op1);
13236 case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op1);
13237 default: return 0;
13238 }
13239}
13240
13241// FastEmit functions for X86ISD::BT.
13242
13243unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13244 if (RetVT.SimpleTy != MVT::i32)
13245 return 0;
13246 return fastEmitInst_rr(MachineInstOpcode: X86::BT16rr, RC: &X86::GR16RegClass, Op0, Op1);
13247}
13248
13249unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13250 if (RetVT.SimpleTy != MVT::i32)
13251 return 0;
13252 return fastEmitInst_rr(MachineInstOpcode: X86::BT32rr, RC: &X86::GR32RegClass, Op0, Op1);
13253}
13254
13255unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13256 if (RetVT.SimpleTy != MVT::i32)
13257 return 0;
13258 return fastEmitInst_rr(MachineInstOpcode: X86::BT64rr, RC: &X86::GR64RegClass, Op0, Op1);
13259}
13260
13261unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13262 switch (VT.SimpleTy) {
13263 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op1);
13264 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op1);
13265 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op1);
13266 default: return 0;
13267 }
13268}
13269
13270// FastEmit functions for X86ISD::BZHI.
13271
13272unsigned fastEmit_X86ISD_BZHI_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13273 if (RetVT.SimpleTy != MVT::i32)
13274 return 0;
13275 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
13276 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
13277 }
13278 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
13279 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr, RC: &X86::GR32RegClass, Op0, Op1);
13280 }
13281 return 0;
13282}
13283
13284unsigned fastEmit_X86ISD_BZHI_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13285 if (RetVT.SimpleTy != MVT::i64)
13286 return 0;
13287 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
13288 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
13289 }
13290 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
13291 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr, RC: &X86::GR64RegClass, Op0, Op1);
13292 }
13293 return 0;
13294}
13295
13296unsigned fastEmit_X86ISD_BZHI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13297 switch (VT.SimpleTy) {
13298 case MVT::i32: return fastEmit_X86ISD_BZHI_MVT_i32_rr(RetVT, Op0, Op1);
13299 case MVT::i64: return fastEmit_X86ISD_BZHI_MVT_i64_rr(RetVT, Op0, Op1);
13300 default: return 0;
13301 }
13302}
13303
13304// FastEmit functions for X86ISD::CMP.
13305
13306unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13307 if (RetVT.SimpleTy != MVT::i32)
13308 return 0;
13309 return fastEmitInst_rr(MachineInstOpcode: X86::CMP8rr, RC: &X86::GR8RegClass, Op0, Op1);
13310}
13311
13312unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13313 if (RetVT.SimpleTy != MVT::i32)
13314 return 0;
13315 return fastEmitInst_rr(MachineInstOpcode: X86::CMP16rr, RC: &X86::GR16RegClass, Op0, Op1);
13316}
13317
13318unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13319 if (RetVT.SimpleTy != MVT::i32)
13320 return 0;
13321 return fastEmitInst_rr(MachineInstOpcode: X86::CMP32rr, RC: &X86::GR32RegClass, Op0, Op1);
13322}
13323
13324unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13325 if (RetVT.SimpleTy != MVT::i32)
13326 return 0;
13327 return fastEmitInst_rr(MachineInstOpcode: X86::CMP64rr, RC: &X86::GR64RegClass, Op0, Op1);
13328}
13329
13330unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13331 switch (VT.SimpleTy) {
13332 case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op1);
13333 case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op1);
13334 case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
13335 case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op1);
13336 default: return 0;
13337 }
13338}
13339
13340// FastEmit functions for X86ISD::COMI.
13341
13342unsigned fastEmit_X86ISD_COMI_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13343 if (RetVT.SimpleTy != MVT::i32)
13344 return 0;
13345 if ((Subtarget->hasFP16())) {
13346 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
13347 }
13348 return 0;
13349}
13350
13351unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13352 if (RetVT.SimpleTy != MVT::i32)
13353 return 0;
13354 if ((Subtarget->hasAVX512())) {
13355 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
13356 }
13357 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13358 return fastEmitInst_rr(MachineInstOpcode: X86::COMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
13359 }
13360 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13361 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
13362 }
13363 return 0;
13364}
13365
13366unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13367 if (RetVT.SimpleTy != MVT::i32)
13368 return 0;
13369 if ((Subtarget->hasAVX512())) {
13370 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
13371 }
13372 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13373 return fastEmitInst_rr(MachineInstOpcode: X86::COMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
13374 }
13375 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13376 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
13377 }
13378 return 0;
13379}
13380
13381unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13382 switch (VT.SimpleTy) {
13383 case MVT::v8f16: return fastEmit_X86ISD_COMI_MVT_v8f16_rr(RetVT, Op0, Op1);
13384 case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op1);
13385 case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op1);
13386 default: return 0;
13387 }
13388}
13389
13390// FastEmit functions for X86ISD::CVTNE2PS2BF16.
13391
13392unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13393 if (RetVT.SimpleTy != MVT::v8bf16)
13394 return 0;
13395 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
13396 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13397 }
13398 return 0;
13399}
13400
13401unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13402 if (RetVT.SimpleTy != MVT::v16bf16)
13403 return 0;
13404 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
13405 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13406 }
13407 return 0;
13408}
13409
13410unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13411 if (RetVT.SimpleTy != MVT::v32bf16)
13412 return 0;
13413 if ((Subtarget->hasBF16())) {
13414 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
13415 }
13416 return 0;
13417}
13418
13419unsigned fastEmit_X86ISD_CVTNE2PS2BF16_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13420 switch (VT.SimpleTy) {
13421 case MVT::v4f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v4f32_rr(RetVT, Op0, Op1);
13422 case MVT::v8f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v8f32_rr(RetVT, Op0, Op1);
13423 case MVT::v16f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v16f32_rr(RetVT, Op0, Op1);
13424 default: return 0;
13425 }
13426}
13427
13428// FastEmit functions for X86ISD::FADDS.
13429
13430unsigned fastEmit_X86ISD_FADDS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13431 if (RetVT.SimpleTy != MVT::v8f16)
13432 return 0;
13433 if ((Subtarget->hasFP16())) {
13434 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
13435 }
13436 return 0;
13437}
13438
13439unsigned fastEmit_X86ISD_FADDS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13440 if (RetVT.SimpleTy != MVT::v4f32)
13441 return 0;
13442 if ((Subtarget->hasAVX512())) {
13443 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
13444 }
13445 return 0;
13446}
13447
13448unsigned fastEmit_X86ISD_FADDS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13449 if (RetVT.SimpleTy != MVT::v2f64)
13450 return 0;
13451 if ((Subtarget->hasAVX512())) {
13452 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
13453 }
13454 return 0;
13455}
13456
13457unsigned fastEmit_X86ISD_FADDS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13458 switch (VT.SimpleTy) {
13459 case MVT::v8f16: return fastEmit_X86ISD_FADDS_MVT_v8f16_rr(RetVT, Op0, Op1);
13460 case MVT::v4f32: return fastEmit_X86ISD_FADDS_MVT_v4f32_rr(RetVT, Op0, Op1);
13461 case MVT::v2f64: return fastEmit_X86ISD_FADDS_MVT_v2f64_rr(RetVT, Op0, Op1);
13462 default: return 0;
13463 }
13464}
13465
13466// FastEmit functions for X86ISD::FAND.
13467
13468unsigned fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13469 if (RetVT.SimpleTy != MVT::f128)
13470 return 0;
13471 if ((Subtarget->hasVLX())) {
13472 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13473 }
13474 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13475 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13476 }
13477 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13478 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13479 }
13480 return 0;
13481}
13482
13483unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13484 if (RetVT.SimpleTy != MVT::v4f32)
13485 return 0;
13486 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13487}
13488
13489unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13490 switch (VT.SimpleTy) {
13491 case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op1);
13492 case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op1);
13493 default: return 0;
13494 }
13495}
13496
13497// FastEmit functions for X86ISD::FANDN.
13498
13499unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13500 if (RetVT.SimpleTy != MVT::v4f32)
13501 return 0;
13502 return fastEmitInst_rr(MachineInstOpcode: X86::ANDNPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13503}
13504
13505unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13506 switch (VT.SimpleTy) {
13507 case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op1);
13508 default: return 0;
13509 }
13510}
13511
13512// FastEmit functions for X86ISD::FCMP.
13513
13514unsigned fastEmit_X86ISD_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13515 if (RetVT.SimpleTy != MVT::i32)
13516 return 0;
13517 if ((Subtarget->hasFP16())) {
13518 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13519 }
13520 return 0;
13521}
13522
13523unsigned fastEmit_X86ISD_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13524 if (RetVT.SimpleTy != MVT::i32)
13525 return 0;
13526 if ((Subtarget->hasAVX512())) {
13527 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13528 }
13529 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13530 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
13531 }
13532 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13533 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
13534 }
13535 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
13536 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
13537 }
13538 return 0;
13539}
13540
13541unsigned fastEmit_X86ISD_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13542 if (RetVT.SimpleTy != MVT::i32)
13543 return 0;
13544 if ((Subtarget->hasAVX512())) {
13545 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13546 }
13547 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13548 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
13549 }
13550 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13551 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
13552 }
13553 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
13554 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
13555 }
13556 return 0;
13557}
13558
13559unsigned fastEmit_X86ISD_FCMP_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13560 if (RetVT.SimpleTy != MVT::i32)
13561 return 0;
13562 if ((Subtarget->canUseCMOV())) {
13563 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
13564 }
13565 return 0;
13566}
13567
13568unsigned fastEmit_X86ISD_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13569 switch (VT.SimpleTy) {
13570 case MVT::f16: return fastEmit_X86ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
13571 case MVT::f32: return fastEmit_X86ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
13572 case MVT::f64: return fastEmit_X86ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
13573 case MVT::f80: return fastEmit_X86ISD_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
13574 default: return 0;
13575 }
13576}
13577
13578// FastEmit functions for X86ISD::FDIVS.
13579
13580unsigned fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13581 if (RetVT.SimpleTy != MVT::v8f16)
13582 return 0;
13583 if ((Subtarget->hasFP16())) {
13584 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
13585 }
13586 return 0;
13587}
13588
13589unsigned fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13590 if (RetVT.SimpleTy != MVT::v4f32)
13591 return 0;
13592 if ((Subtarget->hasAVX512())) {
13593 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
13594 }
13595 return 0;
13596}
13597
13598unsigned fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13599 if (RetVT.SimpleTy != MVT::v2f64)
13600 return 0;
13601 if ((Subtarget->hasAVX512())) {
13602 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
13603 }
13604 return 0;
13605}
13606
13607unsigned fastEmit_X86ISD_FDIVS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13608 switch (VT.SimpleTy) {
13609 case MVT::v8f16: return fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(RetVT, Op0, Op1);
13610 case MVT::v4f32: return fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(RetVT, Op0, Op1);
13611 case MVT::v2f64: return fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(RetVT, Op0, Op1);
13612 default: return 0;
13613 }
13614}
13615
13616// FastEmit functions for X86ISD::FGETEXPS.
13617
13618unsigned fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13619 if (RetVT.SimpleTy != MVT::v8f16)
13620 return 0;
13621 if ((Subtarget->hasFP16())) {
13622 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZr, RC: &X86::VR128XRegClass, Op0, Op1);
13623 }
13624 return 0;
13625}
13626
13627unsigned fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13628 if (RetVT.SimpleTy != MVT::v4f32)
13629 return 0;
13630 if ((Subtarget->hasAVX512())) {
13631 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZr, RC: &X86::VR128XRegClass, Op0, Op1);
13632 }
13633 return 0;
13634}
13635
13636unsigned fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13637 if (RetVT.SimpleTy != MVT::v2f64)
13638 return 0;
13639 if ((Subtarget->hasAVX512())) {
13640 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZr, RC: &X86::VR128XRegClass, Op0, Op1);
13641 }
13642 return 0;
13643}
13644
13645unsigned fastEmit_X86ISD_FGETEXPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13646 switch (VT.SimpleTy) {
13647 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(RetVT, Op0, Op1);
13648 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(RetVT, Op0, Op1);
13649 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(RetVT, Op0, Op1);
13650 default: return 0;
13651 }
13652}
13653
13654// FastEmit functions for X86ISD::FGETEXPS_SAE.
13655
13656unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13657 if (RetVT.SimpleTy != MVT::v8f16)
13658 return 0;
13659 if ((Subtarget->hasFP16())) {
13660 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZrb, RC: &X86::VR128XRegClass, Op0, Op1);
13661 }
13662 return 0;
13663}
13664
13665unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13666 if (RetVT.SimpleTy != MVT::v4f32)
13667 return 0;
13668 if ((Subtarget->hasAVX512())) {
13669 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZrb, RC: &X86::VR128XRegClass, Op0, Op1);
13670 }
13671 return 0;
13672}
13673
13674unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13675 if (RetVT.SimpleTy != MVT::v2f64)
13676 return 0;
13677 if ((Subtarget->hasAVX512())) {
13678 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZrb, RC: &X86::VR128XRegClass, Op0, Op1);
13679 }
13680 return 0;
13681}
13682
13683unsigned fastEmit_X86ISD_FGETEXPS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13684 switch (VT.SimpleTy) {
13685 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
13686 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
13687 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
13688 default: return 0;
13689 }
13690}
13691
13692// FastEmit functions for X86ISD::FHADD.
13693
13694unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13695 if (RetVT.SimpleTy != MVT::v4f32)
13696 return 0;
13697 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
13698 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13699 }
13700 if ((Subtarget->hasAVX())) {
13701 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13702 }
13703 return 0;
13704}
13705
13706unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13707 if (RetVT.SimpleTy != MVT::v8f32)
13708 return 0;
13709 if ((Subtarget->hasAVX())) {
13710 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13711 }
13712 return 0;
13713}
13714
13715unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13716 if (RetVT.SimpleTy != MVT::v2f64)
13717 return 0;
13718 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
13719 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13720 }
13721 if ((Subtarget->hasAVX())) {
13722 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13723 }
13724 return 0;
13725}
13726
13727unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13728 if (RetVT.SimpleTy != MVT::v4f64)
13729 return 0;
13730 if ((Subtarget->hasAVX())) {
13731 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13732 }
13733 return 0;
13734}
13735
13736unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13737 switch (VT.SimpleTy) {
13738 case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op1);
13739 case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op1);
13740 case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op1);
13741 case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op1);
13742 default: return 0;
13743 }
13744}
13745
13746// FastEmit functions for X86ISD::FHSUB.
13747
13748unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13749 if (RetVT.SimpleTy != MVT::v4f32)
13750 return 0;
13751 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
13752 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13753 }
13754 if ((Subtarget->hasAVX())) {
13755 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13756 }
13757 return 0;
13758}
13759
13760unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13761 if (RetVT.SimpleTy != MVT::v8f32)
13762 return 0;
13763 if ((Subtarget->hasAVX())) {
13764 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13765 }
13766 return 0;
13767}
13768
13769unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13770 if (RetVT.SimpleTy != MVT::v2f64)
13771 return 0;
13772 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
13773 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13774 }
13775 if ((Subtarget->hasAVX())) {
13776 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13777 }
13778 return 0;
13779}
13780
13781unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13782 if (RetVT.SimpleTy != MVT::v4f64)
13783 return 0;
13784 if ((Subtarget->hasAVX())) {
13785 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13786 }
13787 return 0;
13788}
13789
13790unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13791 switch (VT.SimpleTy) {
13792 case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13793 case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
13794 case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13795 case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
13796 default: return 0;
13797 }
13798}
13799
13800// FastEmit functions for X86ISD::FMAX.
13801
13802unsigned fastEmit_X86ISD_FMAX_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13803 if (RetVT.SimpleTy != MVT::f16)
13804 return 0;
13805 if ((Subtarget->hasFP16())) {
13806 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13807 }
13808 return 0;
13809}
13810
13811unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13812 if (RetVT.SimpleTy != MVT::f32)
13813 return 0;
13814 if ((Subtarget->hasAVX512())) {
13815 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13816 }
13817 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13818 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13819 }
13820 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13821 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13822 }
13823 return 0;
13824}
13825
13826unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13827 if (RetVT.SimpleTy != MVT::f64)
13828 return 0;
13829 if ((Subtarget->hasAVX512())) {
13830 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13831 }
13832 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13833 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13834 }
13835 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13836 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13837 }
13838 return 0;
13839}
13840
13841unsigned fastEmit_X86ISD_FMAX_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13842 if (RetVT.SimpleTy != MVT::v8f16)
13843 return 0;
13844 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13845 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13846 }
13847 return 0;
13848}
13849
13850unsigned fastEmit_X86ISD_FMAX_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13851 if (RetVT.SimpleTy != MVT::v16f16)
13852 return 0;
13853 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13854 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13855 }
13856 return 0;
13857}
13858
13859unsigned fastEmit_X86ISD_FMAX_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13860 if (RetVT.SimpleTy != MVT::v32f16)
13861 return 0;
13862 if ((Subtarget->hasFP16())) {
13863 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13864 }
13865 return 0;
13866}
13867
13868unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13869 if (RetVT.SimpleTy != MVT::v4f32)
13870 return 0;
13871 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13872 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13873 }
13874 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13875 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13876 }
13877 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13878 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13879 }
13880 return 0;
13881}
13882
13883unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13884 if (RetVT.SimpleTy != MVT::v8f32)
13885 return 0;
13886 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13887 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13888 }
13889 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13890 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13891 }
13892 return 0;
13893}
13894
13895unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13896 if (RetVT.SimpleTy != MVT::v16f32)
13897 return 0;
13898 if ((Subtarget->hasAVX512())) {
13899 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13900 }
13901 return 0;
13902}
13903
13904unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13905 if (RetVT.SimpleTy != MVT::v2f64)
13906 return 0;
13907 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13908 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13909 }
13910 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13911 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13912 }
13913 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13914 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13915 }
13916 return 0;
13917}
13918
13919unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13920 if (RetVT.SimpleTy != MVT::v4f64)
13921 return 0;
13922 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13923 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13924 }
13925 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13926 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13927 }
13928 return 0;
13929}
13930
13931unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13932 if (RetVT.SimpleTy != MVT::v8f64)
13933 return 0;
13934 if ((Subtarget->hasAVX512())) {
13935 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13936 }
13937 return 0;
13938}
13939
13940unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13941 switch (VT.SimpleTy) {
13942 case MVT::f16: return fastEmit_X86ISD_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
13943 case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
13944 case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
13945 case MVT::v8f16: return fastEmit_X86ISD_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
13946 case MVT::v16f16: return fastEmit_X86ISD_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
13947 case MVT::v32f16: return fastEmit_X86ISD_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
13948 case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
13949 case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
13950 case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
13951 case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
13952 case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
13953 case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
13954 default: return 0;
13955 }
13956}
13957
13958// FastEmit functions for X86ISD::FMAXC.
13959
13960unsigned fastEmit_X86ISD_FMAXC_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13961 if (RetVT.SimpleTy != MVT::f16)
13962 return 0;
13963 if ((Subtarget->hasAVX512())) {
13964 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13965 }
13966 return 0;
13967}
13968
13969unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13970 if (RetVT.SimpleTy != MVT::f32)
13971 return 0;
13972 if ((Subtarget->hasAVX512())) {
13973 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13974 }
13975 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13976 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13977 }
13978 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13979 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13980 }
13981 return 0;
13982}
13983
13984unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13985 if (RetVT.SimpleTy != MVT::f64)
13986 return 0;
13987 if ((Subtarget->hasAVX512())) {
13988 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13989 }
13990 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13991 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13992 }
13993 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13994 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13995 }
13996 return 0;
13997}
13998
13999unsigned fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14000 if (RetVT.SimpleTy != MVT::v8f16)
14001 return 0;
14002 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14003 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14004 }
14005 return 0;
14006}
14007
14008unsigned fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14009 if (RetVT.SimpleTy != MVT::v16f16)
14010 return 0;
14011 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14012 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14013 }
14014 return 0;
14015}
14016
14017unsigned fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14018 if (RetVT.SimpleTy != MVT::v32f16)
14019 return 0;
14020 if ((Subtarget->hasFP16())) {
14021 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
14022 }
14023 return 0;
14024}
14025
14026unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14027 if (RetVT.SimpleTy != MVT::v4f32)
14028 return 0;
14029 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14030 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14031 }
14032 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14033 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14034 }
14035 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14036 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14037 }
14038 return 0;
14039}
14040
14041unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14042 if (RetVT.SimpleTy != MVT::v8f32)
14043 return 0;
14044 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14045 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14046 }
14047 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14048 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14049 }
14050 return 0;
14051}
14052
14053unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14054 if (RetVT.SimpleTy != MVT::v16f32)
14055 return 0;
14056 if ((Subtarget->hasAVX512())) {
14057 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14058 }
14059 return 0;
14060}
14061
14062unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14063 if (RetVT.SimpleTy != MVT::v2f64)
14064 return 0;
14065 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14066 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14067 }
14068 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14069 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14070 }
14071 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14072 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14073 }
14074 return 0;
14075}
14076
14077unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14078 if (RetVT.SimpleTy != MVT::v4f64)
14079 return 0;
14080 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14081 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14082 }
14083 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14084 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14085 }
14086 return 0;
14087}
14088
14089unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14090 if (RetVT.SimpleTy != MVT::v8f64)
14091 return 0;
14092 if ((Subtarget->hasAVX512())) {
14093 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14094 }
14095 return 0;
14096}
14097
14098unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14099 switch (VT.SimpleTy) {
14100 case MVT::f16: return fastEmit_X86ISD_FMAXC_MVT_f16_rr(RetVT, Op0, Op1);
14101 case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op1);
14102 case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op1);
14103 case MVT::v8f16: return fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(RetVT, Op0, Op1);
14104 case MVT::v16f16: return fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(RetVT, Op0, Op1);
14105 case MVT::v32f16: return fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(RetVT, Op0, Op1);
14106 case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op1);
14107 case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op1);
14108 case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op1);
14109 case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op1);
14110 case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op1);
14111 case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op1);
14112 default: return 0;
14113 }
14114}
14115
14116// FastEmit functions for X86ISD::FMAXS.
14117
14118unsigned fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14119 if (RetVT.SimpleTy != MVT::v8f16)
14120 return 0;
14121 if ((Subtarget->hasFP16())) {
14122 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14123 }
14124 return 0;
14125}
14126
14127unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14128 if (RetVT.SimpleTy != MVT::v4f32)
14129 return 0;
14130 if ((Subtarget->hasAVX512())) {
14131 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14132 }
14133 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14134 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
14135 }
14136 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14137 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
14138 }
14139 return 0;
14140}
14141
14142unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14143 if (RetVT.SimpleTy != MVT::v2f64)
14144 return 0;
14145 if ((Subtarget->hasAVX512())) {
14146 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14147 }
14148 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14149 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
14150 }
14151 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14152 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
14153 }
14154 return 0;
14155}
14156
14157unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14158 switch (VT.SimpleTy) {
14159 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(RetVT, Op0, Op1);
14160 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op1);
14161 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op1);
14162 default: return 0;
14163 }
14164}
14165
14166// FastEmit functions for X86ISD::FMAXS_SAE.
14167
14168unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14169 if (RetVT.SimpleTy != MVT::v8f16)
14170 return 0;
14171 if ((Subtarget->hasFP16())) {
14172 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14173 }
14174 return 0;
14175}
14176
14177unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14178 if (RetVT.SimpleTy != MVT::v4f32)
14179 return 0;
14180 if ((Subtarget->hasAVX512())) {
14181 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14182 }
14183 return 0;
14184}
14185
14186unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14187 if (RetVT.SimpleTy != MVT::v2f64)
14188 return 0;
14189 if ((Subtarget->hasAVX512())) {
14190 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14191 }
14192 return 0;
14193}
14194
14195unsigned fastEmit_X86ISD_FMAXS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14196 switch (VT.SimpleTy) {
14197 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
14198 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
14199 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
14200 default: return 0;
14201 }
14202}
14203
14204// FastEmit functions for X86ISD::FMAX_SAE.
14205
14206unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14207 if (RetVT.SimpleTy != MVT::v32f16)
14208 return 0;
14209 if ((Subtarget->hasFP16())) {
14210 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
14211 }
14212 return 0;
14213}
14214
14215unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14216 if (RetVT.SimpleTy != MVT::v16f32)
14217 return 0;
14218 if ((Subtarget->hasAVX512())) {
14219 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
14220 }
14221 return 0;
14222}
14223
14224unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14225 if (RetVT.SimpleTy != MVT::v8f64)
14226 return 0;
14227 if ((Subtarget->hasAVX512())) {
14228 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
14229 }
14230 return 0;
14231}
14232
14233unsigned fastEmit_X86ISD_FMAX_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14234 switch (VT.SimpleTy) {
14235 case MVT::v32f16: return fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
14236 case MVT::v16f32: return fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
14237 case MVT::v8f64: return fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
14238 default: return 0;
14239 }
14240}
14241
14242// FastEmit functions for X86ISD::FMIN.
14243
14244unsigned fastEmit_X86ISD_FMIN_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14245 if (RetVT.SimpleTy != MVT::f16)
14246 return 0;
14247 if ((Subtarget->hasFP16())) {
14248 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
14249 }
14250 return 0;
14251}
14252
14253unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14254 if (RetVT.SimpleTy != MVT::f32)
14255 return 0;
14256 if ((Subtarget->hasAVX512())) {
14257 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
14258 }
14259 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14260 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14261 }
14262 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14263 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14264 }
14265 return 0;
14266}
14267
14268unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14269 if (RetVT.SimpleTy != MVT::f64)
14270 return 0;
14271 if ((Subtarget->hasAVX512())) {
14272 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
14273 }
14274 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14275 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14276 }
14277 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14278 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14279 }
14280 return 0;
14281}
14282
14283unsigned fastEmit_X86ISD_FMIN_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14284 if (RetVT.SimpleTy != MVT::v8f16)
14285 return 0;
14286 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14287 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14288 }
14289 return 0;
14290}
14291
14292unsigned fastEmit_X86ISD_FMIN_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14293 if (RetVT.SimpleTy != MVT::v16f16)
14294 return 0;
14295 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14296 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14297 }
14298 return 0;
14299}
14300
14301unsigned fastEmit_X86ISD_FMIN_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14302 if (RetVT.SimpleTy != MVT::v32f16)
14303 return 0;
14304 if ((Subtarget->hasFP16())) {
14305 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
14306 }
14307 return 0;
14308}
14309
14310unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14311 if (RetVT.SimpleTy != MVT::v4f32)
14312 return 0;
14313 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14314 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14315 }
14316 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14317 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14318 }
14319 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14320 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14321 }
14322 return 0;
14323}
14324
14325unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14326 if (RetVT.SimpleTy != MVT::v8f32)
14327 return 0;
14328 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14329 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14330 }
14331 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14332 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14333 }
14334 return 0;
14335}
14336
14337unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14338 if (RetVT.SimpleTy != MVT::v16f32)
14339 return 0;
14340 if ((Subtarget->hasAVX512())) {
14341 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14342 }
14343 return 0;
14344}
14345
14346unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14347 if (RetVT.SimpleTy != MVT::v2f64)
14348 return 0;
14349 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14350 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14351 }
14352 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14353 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14354 }
14355 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14356 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14357 }
14358 return 0;
14359}
14360
14361unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14362 if (RetVT.SimpleTy != MVT::v4f64)
14363 return 0;
14364 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14365 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14366 }
14367 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14368 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14369 }
14370 return 0;
14371}
14372
14373unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14374 if (RetVT.SimpleTy != MVT::v8f64)
14375 return 0;
14376 if ((Subtarget->hasAVX512())) {
14377 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14378 }
14379 return 0;
14380}
14381
14382unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14383 switch (VT.SimpleTy) {
14384 case MVT::f16: return fastEmit_X86ISD_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
14385 case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
14386 case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
14387 case MVT::v8f16: return fastEmit_X86ISD_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
14388 case MVT::v16f16: return fastEmit_X86ISD_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
14389 case MVT::v32f16: return fastEmit_X86ISD_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
14390 case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
14391 case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
14392 case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
14393 case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
14394 case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
14395 case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
14396 default: return 0;
14397 }
14398}
14399
14400// FastEmit functions for X86ISD::FMINC.
14401
14402unsigned fastEmit_X86ISD_FMINC_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14403 if (RetVT.SimpleTy != MVT::f16)
14404 return 0;
14405 if ((Subtarget->hasAVX512())) {
14406 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
14407 }
14408 return 0;
14409}
14410
14411unsigned fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14412 if (RetVT.SimpleTy != MVT::f32)
14413 return 0;
14414 if ((Subtarget->hasAVX512())) {
14415 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
14416 }
14417 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14418 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14419 }
14420 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14421 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14422 }
14423 return 0;
14424}
14425
14426unsigned fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14427 if (RetVT.SimpleTy != MVT::f64)
14428 return 0;
14429 if ((Subtarget->hasAVX512())) {
14430 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
14431 }
14432 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14433 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14434 }
14435 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14436 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14437 }
14438 return 0;
14439}
14440
14441unsigned fastEmit_X86ISD_FMINC_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14442 if (RetVT.SimpleTy != MVT::v8f16)
14443 return 0;
14444 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14445 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14446 }
14447 return 0;
14448}
14449
14450unsigned fastEmit_X86ISD_FMINC_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14451 if (RetVT.SimpleTy != MVT::v16f16)
14452 return 0;
14453 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14454 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14455 }
14456 return 0;
14457}
14458
14459unsigned fastEmit_X86ISD_FMINC_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14460 if (RetVT.SimpleTy != MVT::v32f16)
14461 return 0;
14462 if ((Subtarget->hasFP16())) {
14463 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
14464 }
14465 return 0;
14466}
14467
14468unsigned fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14469 if (RetVT.SimpleTy != MVT::v4f32)
14470 return 0;
14471 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14472 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14473 }
14474 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14475 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14476 }
14477 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14478 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14479 }
14480 return 0;
14481}
14482
14483unsigned fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14484 if (RetVT.SimpleTy != MVT::v8f32)
14485 return 0;
14486 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14487 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14488 }
14489 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14490 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14491 }
14492 return 0;
14493}
14494
14495unsigned fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14496 if (RetVT.SimpleTy != MVT::v16f32)
14497 return 0;
14498 if ((Subtarget->hasAVX512())) {
14499 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14500 }
14501 return 0;
14502}
14503
14504unsigned fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14505 if (RetVT.SimpleTy != MVT::v2f64)
14506 return 0;
14507 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14508 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14509 }
14510 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14511 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14512 }
14513 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14514 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14515 }
14516 return 0;
14517}
14518
14519unsigned fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14520 if (RetVT.SimpleTy != MVT::v4f64)
14521 return 0;
14522 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14523 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14524 }
14525 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14526 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14527 }
14528 return 0;
14529}
14530
14531unsigned fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14532 if (RetVT.SimpleTy != MVT::v8f64)
14533 return 0;
14534 if ((Subtarget->hasAVX512())) {
14535 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14536 }
14537 return 0;
14538}
14539
14540unsigned fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14541 switch (VT.SimpleTy) {
14542 case MVT::f16: return fastEmit_X86ISD_FMINC_MVT_f16_rr(RetVT, Op0, Op1);
14543 case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op1);
14544 case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op1);
14545 case MVT::v8f16: return fastEmit_X86ISD_FMINC_MVT_v8f16_rr(RetVT, Op0, Op1);
14546 case MVT::v16f16: return fastEmit_X86ISD_FMINC_MVT_v16f16_rr(RetVT, Op0, Op1);
14547 case MVT::v32f16: return fastEmit_X86ISD_FMINC_MVT_v32f16_rr(RetVT, Op0, Op1);
14548 case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op1);
14549 case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op1);
14550 case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op1);
14551 case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op1);
14552 case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op1);
14553 case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op1);
14554 default: return 0;
14555 }
14556}
14557
14558// FastEmit functions for X86ISD::FMINS.
14559
14560unsigned fastEmit_X86ISD_FMINS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14561 if (RetVT.SimpleTy != MVT::v8f16)
14562 return 0;
14563 if ((Subtarget->hasFP16())) {
14564 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14565 }
14566 return 0;
14567}
14568
14569unsigned fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14570 if (RetVT.SimpleTy != MVT::v4f32)
14571 return 0;
14572 if ((Subtarget->hasAVX512())) {
14573 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14574 }
14575 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14576 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
14577 }
14578 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14579 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
14580 }
14581 return 0;
14582}
14583
14584unsigned fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14585 if (RetVT.SimpleTy != MVT::v2f64)
14586 return 0;
14587 if ((Subtarget->hasAVX512())) {
14588 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14589 }
14590 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14591 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
14592 }
14593 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14594 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
14595 }
14596 return 0;
14597}
14598
14599unsigned fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14600 switch (VT.SimpleTy) {
14601 case MVT::v8f16: return fastEmit_X86ISD_FMINS_MVT_v8f16_rr(RetVT, Op0, Op1);
14602 case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op1);
14603 case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op1);
14604 default: return 0;
14605 }
14606}
14607
14608// FastEmit functions for X86ISD::FMINS_SAE.
14609
14610unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14611 if (RetVT.SimpleTy != MVT::v8f16)
14612 return 0;
14613 if ((Subtarget->hasFP16())) {
14614 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14615 }
14616 return 0;
14617}
14618
14619unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14620 if (RetVT.SimpleTy != MVT::v4f32)
14621 return 0;
14622 if ((Subtarget->hasAVX512())) {
14623 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14624 }
14625 return 0;
14626}
14627
14628unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14629 if (RetVT.SimpleTy != MVT::v2f64)
14630 return 0;
14631 if ((Subtarget->hasAVX512())) {
14632 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14633 }
14634 return 0;
14635}
14636
14637unsigned fastEmit_X86ISD_FMINS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14638 switch (VT.SimpleTy) {
14639 case MVT::v8f16: return fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
14640 case MVT::v4f32: return fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
14641 case MVT::v2f64: return fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
14642 default: return 0;
14643 }
14644}
14645
14646// FastEmit functions for X86ISD::FMIN_SAE.
14647
14648unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14649 if (RetVT.SimpleTy != MVT::v32f16)
14650 return 0;
14651 if ((Subtarget->hasFP16())) {
14652 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
14653 }
14654 return 0;
14655}
14656
14657unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14658 if (RetVT.SimpleTy != MVT::v16f32)
14659 return 0;
14660 if ((Subtarget->hasAVX512())) {
14661 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
14662 }
14663 return 0;
14664}
14665
14666unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14667 if (RetVT.SimpleTy != MVT::v8f64)
14668 return 0;
14669 if ((Subtarget->hasAVX512())) {
14670 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
14671 }
14672 return 0;
14673}
14674
14675unsigned fastEmit_X86ISD_FMIN_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14676 switch (VT.SimpleTy) {
14677 case MVT::v32f16: return fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
14678 case MVT::v16f32: return fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
14679 case MVT::v8f64: return fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
14680 default: return 0;
14681 }
14682}
14683
14684// FastEmit functions for X86ISD::FMULS.
14685
14686unsigned fastEmit_X86ISD_FMULS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14687 if (RetVT.SimpleTy != MVT::v8f16)
14688 return 0;
14689 if ((Subtarget->hasFP16())) {
14690 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14691 }
14692 return 0;
14693}
14694
14695unsigned fastEmit_X86ISD_FMULS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14696 if (RetVT.SimpleTy != MVT::v4f32)
14697 return 0;
14698 if ((Subtarget->hasAVX512())) {
14699 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14700 }
14701 return 0;
14702}
14703
14704unsigned fastEmit_X86ISD_FMULS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14705 if (RetVT.SimpleTy != MVT::v2f64)
14706 return 0;
14707 if ((Subtarget->hasAVX512())) {
14708 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14709 }
14710 return 0;
14711}
14712
14713unsigned fastEmit_X86ISD_FMULS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14714 switch (VT.SimpleTy) {
14715 case MVT::v8f16: return fastEmit_X86ISD_FMULS_MVT_v8f16_rr(RetVT, Op0, Op1);
14716 case MVT::v4f32: return fastEmit_X86ISD_FMULS_MVT_v4f32_rr(RetVT, Op0, Op1);
14717 case MVT::v2f64: return fastEmit_X86ISD_FMULS_MVT_v2f64_rr(RetVT, Op0, Op1);
14718 default: return 0;
14719 }
14720}
14721
14722// FastEmit functions for X86ISD::FOR.
14723
14724unsigned fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14725 if (RetVT.SimpleTy != MVT::f128)
14726 return 0;
14727 if ((Subtarget->hasVLX())) {
14728 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14729 }
14730 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14731 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14732 }
14733 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14734 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14735 }
14736 return 0;
14737}
14738
14739unsigned fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14740 if (RetVT.SimpleTy != MVT::v4f32)
14741 return 0;
14742 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14743}
14744
14745unsigned fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14746 switch (VT.SimpleTy) {
14747 case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op1);
14748 case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op1);
14749 default: return 0;
14750 }
14751}
14752
14753// FastEmit functions for X86ISD::FP80_ADD.
14754
14755unsigned fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14756 if (RetVT.SimpleTy != MVT::f80)
14757 return 0;
14758 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
14759}
14760
14761unsigned fastEmit_X86ISD_FP80_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14762 switch (VT.SimpleTy) {
14763 case MVT::f80: return fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
14764 default: return 0;
14765 }
14766}
14767
14768// FastEmit functions for X86ISD::FSQRTS.
14769
14770unsigned fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14771 if (RetVT.SimpleTy != MVT::v8f16)
14772 return 0;
14773 if ((Subtarget->hasFP16())) {
14774 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSHZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14775 }
14776 return 0;
14777}
14778
14779unsigned fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14780 if (RetVT.SimpleTy != MVT::v4f32)
14781 return 0;
14782 if ((Subtarget->hasAVX512())) {
14783 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSSZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14784 }
14785 return 0;
14786}
14787
14788unsigned fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14789 if (RetVT.SimpleTy != MVT::v2f64)
14790 return 0;
14791 if ((Subtarget->hasAVX512())) {
14792 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSDZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14793 }
14794 return 0;
14795}
14796
14797unsigned fastEmit_X86ISD_FSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14798 switch (VT.SimpleTy) {
14799 case MVT::v8f16: return fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(RetVT, Op0, Op1);
14800 case MVT::v4f32: return fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
14801 case MVT::v2f64: return fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
14802 default: return 0;
14803 }
14804}
14805
14806// FastEmit functions for X86ISD::FSUBS.
14807
14808unsigned fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14809 if (RetVT.SimpleTy != MVT::v8f16)
14810 return 0;
14811 if ((Subtarget->hasFP16())) {
14812 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14813 }
14814 return 0;
14815}
14816
14817unsigned fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14818 if (RetVT.SimpleTy != MVT::v4f32)
14819 return 0;
14820 if ((Subtarget->hasAVX512())) {
14821 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14822 }
14823 return 0;
14824}
14825
14826unsigned fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14827 if (RetVT.SimpleTy != MVT::v2f64)
14828 return 0;
14829 if ((Subtarget->hasAVX512())) {
14830 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
14831 }
14832 return 0;
14833}
14834
14835unsigned fastEmit_X86ISD_FSUBS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14836 switch (VT.SimpleTy) {
14837 case MVT::v8f16: return fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(RetVT, Op0, Op1);
14838 case MVT::v4f32: return fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(RetVT, Op0, Op1);
14839 case MVT::v2f64: return fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(RetVT, Op0, Op1);
14840 default: return 0;
14841 }
14842}
14843
14844// FastEmit functions for X86ISD::FXOR.
14845
14846unsigned fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14847 if (RetVT.SimpleTy != MVT::f128)
14848 return 0;
14849 if ((Subtarget->hasVLX())) {
14850 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14851 }
14852 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14853 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14854 }
14855 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14856 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14857 }
14858 return 0;
14859}
14860
14861unsigned fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14862 if (RetVT.SimpleTy != MVT::v4f32)
14863 return 0;
14864 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14865}
14866
14867unsigned fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14868 switch (VT.SimpleTy) {
14869 case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op1);
14870 case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op1);
14871 default: return 0;
14872 }
14873}
14874
14875// FastEmit functions for X86ISD::GF2P8MULB.
14876
14877unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14878 if (RetVT.SimpleTy != MVT::v16i8)
14879 return 0;
14880 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
14881 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14882 }
14883 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
14884 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
14885 }
14886 if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14887 return fastEmitInst_rr(MachineInstOpcode: X86::GF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
14888 }
14889 return 0;
14890}
14891
14892unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14893 if (RetVT.SimpleTy != MVT::v32i8)
14894 return 0;
14895 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
14896 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14897 }
14898 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
14899 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14900 }
14901 return 0;
14902}
14903
14904unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14905 if (RetVT.SimpleTy != MVT::v64i8)
14906 return 0;
14907 if ((Subtarget->hasAVX512()) && (Subtarget->hasGFNI())) {
14908 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14909 }
14910 return 0;
14911}
14912
14913unsigned fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14914 switch (VT.SimpleTy) {
14915 case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op1);
14916 case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op1);
14917 case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op1);
14918 default: return 0;
14919 }
14920}
14921
14922// FastEmit functions for X86ISD::HADD.
14923
14924unsigned fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14925 if (RetVT.SimpleTy != MVT::v8i16)
14926 return 0;
14927 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
14928 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
14929 }
14930 if ((Subtarget->hasAVX())) {
14931 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
14932 }
14933 return 0;
14934}
14935
14936unsigned fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14937 if (RetVT.SimpleTy != MVT::v16i16)
14938 return 0;
14939 if ((Subtarget->hasAVX2())) {
14940 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14941 }
14942 return 0;
14943}
14944
14945unsigned fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14946 if (RetVT.SimpleTy != MVT::v4i32)
14947 return 0;
14948 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
14949 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
14950 }
14951 if ((Subtarget->hasAVX())) {
14952 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
14953 }
14954 return 0;
14955}
14956
14957unsigned fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14958 if (RetVT.SimpleTy != MVT::v8i32)
14959 return 0;
14960 if ((Subtarget->hasAVX2())) {
14961 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14962 }
14963 return 0;
14964}
14965
14966unsigned fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14967 switch (VT.SimpleTy) {
14968 case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op1);
14969 case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op1);
14970 case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op1);
14971 case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op1);
14972 default: return 0;
14973 }
14974}
14975
14976// FastEmit functions for X86ISD::HSUB.
14977
14978unsigned fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14979 if (RetVT.SimpleTy != MVT::v8i16)
14980 return 0;
14981 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
14982 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
14983 }
14984 if ((Subtarget->hasAVX())) {
14985 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
14986 }
14987 return 0;
14988}
14989
14990unsigned fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14991 if (RetVT.SimpleTy != MVT::v16i16)
14992 return 0;
14993 if ((Subtarget->hasAVX2())) {
14994 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14995 }
14996 return 0;
14997}
14998
14999unsigned fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15000 if (RetVT.SimpleTy != MVT::v4i32)
15001 return 0;
15002 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
15003 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
15004 }
15005 if ((Subtarget->hasAVX())) {
15006 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
15007 }
15008 return 0;
15009}
15010
15011unsigned fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15012 if (RetVT.SimpleTy != MVT::v8i32)
15013 return 0;
15014 if ((Subtarget->hasAVX2())) {
15015 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15016 }
15017 return 0;
15018}
15019
15020unsigned fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15021 switch (VT.SimpleTy) {
15022 case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op1);
15023 case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op1);
15024 case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op1);
15025 case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op1);
15026 default: return 0;
15027 }
15028}
15029
15030// FastEmit functions for X86ISD::KADD.
15031
15032unsigned fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15033 if (RetVT.SimpleTy != MVT::v8i1)
15034 return 0;
15035 if ((Subtarget->hasDQI())) {
15036 return fastEmitInst_rr(MachineInstOpcode: X86::KADDBrr, RC: &X86::VK8RegClass, Op0, Op1);
15037 }
15038 return 0;
15039}
15040
15041unsigned fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15042 if (RetVT.SimpleTy != MVT::v16i1)
15043 return 0;
15044 if ((Subtarget->hasDQI())) {
15045 return fastEmitInst_rr(MachineInstOpcode: X86::KADDWrr, RC: &X86::VK16RegClass, Op0, Op1);
15046 }
15047 return 0;
15048}
15049
15050unsigned fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15051 if (RetVT.SimpleTy != MVT::v32i1)
15052 return 0;
15053 if ((Subtarget->hasBWI())) {
15054 return fastEmitInst_rr(MachineInstOpcode: X86::KADDDrr, RC: &X86::VK32RegClass, Op0, Op1);
15055 }
15056 return 0;
15057}
15058
15059unsigned fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15060 if (RetVT.SimpleTy != MVT::v64i1)
15061 return 0;
15062 if ((Subtarget->hasBWI())) {
15063 return fastEmitInst_rr(MachineInstOpcode: X86::KADDQrr, RC: &X86::VK64RegClass, Op0, Op1);
15064 }
15065 return 0;
15066}
15067
15068unsigned fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15069 switch (VT.SimpleTy) {
15070 case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op1);
15071 case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op1);
15072 case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op1);
15073 case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op1);
15074 default: return 0;
15075 }
15076}
15077
15078// FastEmit functions for X86ISD::KORTEST.
15079
15080unsigned fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15081 if (RetVT.SimpleTy != MVT::i32)
15082 return 0;
15083 if ((Subtarget->hasDQI())) {
15084 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTBrr, RC: &X86::VK8RegClass, Op0, Op1);
15085 }
15086 return 0;
15087}
15088
15089unsigned fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15090 if (RetVT.SimpleTy != MVT::i32)
15091 return 0;
15092 if ((Subtarget->hasAVX512())) {
15093 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTWrr, RC: &X86::VK16RegClass, Op0, Op1);
15094 }
15095 return 0;
15096}
15097
15098unsigned fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15099 if (RetVT.SimpleTy != MVT::i32)
15100 return 0;
15101 if ((Subtarget->hasBWI())) {
15102 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTDrr, RC: &X86::VK32RegClass, Op0, Op1);
15103 }
15104 return 0;
15105}
15106
15107unsigned fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15108 if (RetVT.SimpleTy != MVT::i32)
15109 return 0;
15110 if ((Subtarget->hasBWI())) {
15111 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTQrr, RC: &X86::VK64RegClass, Op0, Op1);
15112 }
15113 return 0;
15114}
15115
15116unsigned fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15117 switch (VT.SimpleTy) {
15118 case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
15119 case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
15120 case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
15121 case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
15122 default: return 0;
15123 }
15124}
15125
15126// FastEmit functions for X86ISD::KTEST.
15127
15128unsigned fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15129 if (RetVT.SimpleTy != MVT::i32)
15130 return 0;
15131 if ((Subtarget->hasDQI())) {
15132 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTBrr, RC: &X86::VK8RegClass, Op0, Op1);
15133 }
15134 return 0;
15135}
15136
15137unsigned fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15138 if (RetVT.SimpleTy != MVT::i32)
15139 return 0;
15140 if ((Subtarget->hasDQI())) {
15141 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTWrr, RC: &X86::VK16RegClass, Op0, Op1);
15142 }
15143 return 0;
15144}
15145
15146unsigned fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15147 if (RetVT.SimpleTy != MVT::i32)
15148 return 0;
15149 if ((Subtarget->hasBWI())) {
15150 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTDrr, RC: &X86::VK32RegClass, Op0, Op1);
15151 }
15152 return 0;
15153}
15154
15155unsigned fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15156 if (RetVT.SimpleTy != MVT::i32)
15157 return 0;
15158 if ((Subtarget->hasBWI())) {
15159 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTQrr, RC: &X86::VK64RegClass, Op0, Op1);
15160 }
15161 return 0;
15162}
15163
15164unsigned fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15165 switch (VT.SimpleTy) {
15166 case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
15167 case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
15168 case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
15169 case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
15170 default: return 0;
15171 }
15172}
15173
15174// FastEmit functions for X86ISD::MOVHLPS.
15175
15176unsigned fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15177 if (RetVT.SimpleTy != MVT::v4f32)
15178 return 0;
15179 if ((Subtarget->hasAVX512())) {
15180 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
15181 }
15182 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15183 return fastEmitInst_rr(MachineInstOpcode: X86::MOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15184 }
15185 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15186 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15187 }
15188 return 0;
15189}
15190
15191unsigned fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15192 switch (VT.SimpleTy) {
15193 case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op1);
15194 default: return 0;
15195 }
15196}
15197
15198// FastEmit functions for X86ISD::MOVLHPS.
15199
15200unsigned fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15201 if (RetVT.SimpleTy != MVT::v4f32)
15202 return 0;
15203 if ((Subtarget->hasAVX512())) {
15204 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
15205 }
15206 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15207 return fastEmitInst_rr(MachineInstOpcode: X86::MOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15208 }
15209 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15210 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15211 }
15212 return 0;
15213}
15214
15215unsigned fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15216 switch (VT.SimpleTy) {
15217 case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op1);
15218 default: return 0;
15219 }
15220}
15221
15222// FastEmit functions for X86ISD::MOVSD.
15223
15224unsigned fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15225 if (RetVT.SimpleTy != MVT::v2f64)
15226 return 0;
15227 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
15228 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
15229 }
15230 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15231 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
15232 }
15233 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15234 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
15235 }
15236 return 0;
15237}
15238
15239unsigned fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15240 switch (VT.SimpleTy) {
15241 case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op1);
15242 default: return 0;
15243 }
15244}
15245
15246// FastEmit functions for X86ISD::MOVSH.
15247
15248unsigned fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15249 if (RetVT.SimpleTy != MVT::v8f16)
15250 return 0;
15251 if ((Subtarget->hasFP16())) {
15252 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
15253 }
15254 return 0;
15255}
15256
15257unsigned fastEmit_X86ISD_MOVSH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15258 switch (VT.SimpleTy) {
15259 case MVT::v8f16: return fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(RetVT, Op0, Op1);
15260 default: return 0;
15261 }
15262}
15263
15264// FastEmit functions for X86ISD::MOVSS.
15265
15266unsigned fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15267 if (RetVT.SimpleTy != MVT::v4f32)
15268 return 0;
15269 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
15270 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
15271 }
15272 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15273 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
15274 }
15275 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15276 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
15277 }
15278 return 0;
15279}
15280
15281unsigned fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15282 switch (VT.SimpleTy) {
15283 case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op1);
15284 default: return 0;
15285 }
15286}
15287
15288// FastEmit functions for X86ISD::MULHRS.
15289
15290unsigned fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15291 if (RetVT.SimpleTy != MVT::v8i16)
15292 return 0;
15293 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15294 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15295 }
15296 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
15297 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
15298 }
15299 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15300 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
15301 }
15302 return 0;
15303}
15304
15305unsigned fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15306 if (RetVT.SimpleTy != MVT::v16i16)
15307 return 0;
15308 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15309 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15310 }
15311 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15312 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15313 }
15314 return 0;
15315}
15316
15317unsigned fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15318 if (RetVT.SimpleTy != MVT::v32i16)
15319 return 0;
15320 if ((Subtarget->hasBWI())) {
15321 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
15322 }
15323 return 0;
15324}
15325
15326unsigned fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15327 switch (VT.SimpleTy) {
15328 case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op1);
15329 case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op1);
15330 case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op1);
15331 default: return 0;
15332 }
15333}
15334
15335// FastEmit functions for X86ISD::MULTISHIFT.
15336
15337unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15338 if (RetVT.SimpleTy != MVT::v16i8)
15339 return 0;
15340 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
15341 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15342 }
15343 return 0;
15344}
15345
15346unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15347 if (RetVT.SimpleTy != MVT::v32i8)
15348 return 0;
15349 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
15350 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15351 }
15352 return 0;
15353}
15354
15355unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15356 if (RetVT.SimpleTy != MVT::v64i8)
15357 return 0;
15358 if ((Subtarget->hasVBMI())) {
15359 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZrr, RC: &X86::VR512RegClass, Op0, Op1);
15360 }
15361 return 0;
15362}
15363
15364unsigned fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15365 switch (VT.SimpleTy) {
15366 case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op1);
15367 case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op1);
15368 case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op1);
15369 default: return 0;
15370 }
15371}
15372
15373// FastEmit functions for X86ISD::PACKSS.
15374
15375unsigned fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15376 if (RetVT.SimpleTy != MVT::v16i8)
15377 return 0;
15378 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15379 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15380 }
15381 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15382 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
15383 }
15384 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15385 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
15386 }
15387 return 0;
15388}
15389
15390unsigned fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15391 if (RetVT.SimpleTy != MVT::v32i8)
15392 return 0;
15393 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15394 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15395 }
15396 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15397 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
15398 }
15399 return 0;
15400}
15401
15402unsigned fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15403 if (RetVT.SimpleTy != MVT::v64i8)
15404 return 0;
15405 if ((Subtarget->hasBWI())) {
15406 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
15407 }
15408 return 0;
15409}
15410
15411unsigned fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15412 if (RetVT.SimpleTy != MVT::v8i16)
15413 return 0;
15414 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15415 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15416 }
15417 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15418 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
15419 }
15420 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15421 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
15422 }
15423 return 0;
15424}
15425
15426unsigned fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15427 if (RetVT.SimpleTy != MVT::v16i16)
15428 return 0;
15429 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15430 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15431 }
15432 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15433 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15434 }
15435 return 0;
15436}
15437
15438unsigned fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15439 if (RetVT.SimpleTy != MVT::v32i16)
15440 return 0;
15441 if ((Subtarget->hasBWI())) {
15442 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
15443 }
15444 return 0;
15445}
15446
15447unsigned fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15448 switch (VT.SimpleTy) {
15449 case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op1);
15450 case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op1);
15451 case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op1);
15452 case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op1);
15453 case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op1);
15454 case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op1);
15455 default: return 0;
15456 }
15457}
15458
15459// FastEmit functions for X86ISD::PACKUS.
15460
15461unsigned fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15462 if (RetVT.SimpleTy != MVT::v16i8)
15463 return 0;
15464 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15465 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15466 }
15467 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15468 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
15469 }
15470 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15471 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
15472 }
15473 return 0;
15474}
15475
15476unsigned fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15477 if (RetVT.SimpleTy != MVT::v32i8)
15478 return 0;
15479 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15480 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15481 }
15482 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15483 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
15484 }
15485 return 0;
15486}
15487
15488unsigned fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15489 if (RetVT.SimpleTy != MVT::v64i8)
15490 return 0;
15491 if ((Subtarget->hasBWI())) {
15492 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
15493 }
15494 return 0;
15495}
15496
15497unsigned fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15498 if (RetVT.SimpleTy != MVT::v8i16)
15499 return 0;
15500 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15501 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15502 }
15503 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15504 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
15505 }
15506 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15507 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
15508 }
15509 return 0;
15510}
15511
15512unsigned fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15513 if (RetVT.SimpleTy != MVT::v16i16)
15514 return 0;
15515 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15516 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15517 }
15518 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15519 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15520 }
15521 return 0;
15522}
15523
15524unsigned fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15525 if (RetVT.SimpleTy != MVT::v32i16)
15526 return 0;
15527 if ((Subtarget->hasBWI())) {
15528 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
15529 }
15530 return 0;
15531}
15532
15533unsigned fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15534 switch (VT.SimpleTy) {
15535 case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op1);
15536 case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op1);
15537 case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op1);
15538 case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op1);
15539 case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op1);
15540 case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op1);
15541 default: return 0;
15542 }
15543}
15544
15545// FastEmit functions for X86ISD::PCMPEQ.
15546
15547unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15548 if (RetVT.SimpleTy != MVT::v16i8)
15549 return 0;
15550 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15551 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
15552 }
15553 if ((Subtarget->hasAVX()) && (true)) {
15554 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
15555 }
15556 return 0;
15557}
15558
15559unsigned fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15560 if (RetVT.SimpleTy != MVT::v32i8)
15561 return 0;
15562 if ((Subtarget->hasAVX2()) && (true)) {
15563 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBYrr, RC: &X86::VR256RegClass, Op0, Op1);
15564 }
15565 return 0;
15566}
15567
15568unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15569 if (RetVT.SimpleTy != MVT::v8i16)
15570 return 0;
15571 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15572 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
15573 }
15574 if ((Subtarget->hasAVX()) && (true)) {
15575 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
15576 }
15577 return 0;
15578}
15579
15580unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15581 if (RetVT.SimpleTy != MVT::v16i16)
15582 return 0;
15583 if ((Subtarget->hasAVX2()) && (true)) {
15584 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15585 }
15586 return 0;
15587}
15588
15589unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15590 if (RetVT.SimpleTy != MVT::v4i32)
15591 return 0;
15592 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15593 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
15594 }
15595 if ((Subtarget->hasAVX()) && (true)) {
15596 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
15597 }
15598 return 0;
15599}
15600
15601unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15602 if (RetVT.SimpleTy != MVT::v8i32)
15603 return 0;
15604 if ((Subtarget->hasAVX2()) && (true)) {
15605 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15606 }
15607 return 0;
15608}
15609
15610unsigned fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15611 if (RetVT.SimpleTy != MVT::v2i64)
15612 return 0;
15613 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15614 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
15615 }
15616 if ((Subtarget->hasAVX())) {
15617 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
15618 }
15619 return 0;
15620}
15621
15622unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15623 if (RetVT.SimpleTy != MVT::v4i64)
15624 return 0;
15625 if ((Subtarget->hasAVX2())) {
15626 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQYrr, RC: &X86::VR256RegClass, Op0, Op1);
15627 }
15628 return 0;
15629}
15630
15631unsigned fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15632 switch (VT.SimpleTy) {
15633 case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op1);
15634 case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op1);
15635 case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op1);
15636 case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op1);
15637 case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op1);
15638 case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op1);
15639 case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op1);
15640 case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op1);
15641 default: return 0;
15642 }
15643}
15644
15645// FastEmit functions for X86ISD::PCMPGT.
15646
15647unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15648 if (RetVT.SimpleTy != MVT::v16i8)
15649 return 0;
15650 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15651 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
15652 }
15653 if ((Subtarget->hasAVX()) && (true)) {
15654 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
15655 }
15656 return 0;
15657}
15658
15659unsigned fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15660 if (RetVT.SimpleTy != MVT::v32i8)
15661 return 0;
15662 if ((Subtarget->hasAVX2()) && (true)) {
15663 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBYrr, RC: &X86::VR256RegClass, Op0, Op1);
15664 }
15665 return 0;
15666}
15667
15668unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15669 if (RetVT.SimpleTy != MVT::v8i16)
15670 return 0;
15671 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15672 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
15673 }
15674 if ((Subtarget->hasAVX()) && (true)) {
15675 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
15676 }
15677 return 0;
15678}
15679
15680unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15681 if (RetVT.SimpleTy != MVT::v16i16)
15682 return 0;
15683 if ((Subtarget->hasAVX2()) && (true)) {
15684 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15685 }
15686 return 0;
15687}
15688
15689unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15690 if (RetVT.SimpleTy != MVT::v4i32)
15691 return 0;
15692 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15693 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
15694 }
15695 if ((Subtarget->hasAVX()) && (true)) {
15696 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
15697 }
15698 return 0;
15699}
15700
15701unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15702 if (RetVT.SimpleTy != MVT::v8i32)
15703 return 0;
15704 if ((Subtarget->hasAVX2()) && (true)) {
15705 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15706 }
15707 return 0;
15708}
15709
15710unsigned fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15711 if (RetVT.SimpleTy != MVT::v2i64)
15712 return 0;
15713 if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) {
15714 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
15715 }
15716 if ((Subtarget->hasAVX())) {
15717 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
15718 }
15719 return 0;
15720}
15721
15722unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15723 if (RetVT.SimpleTy != MVT::v4i64)
15724 return 0;
15725 if ((Subtarget->hasAVX2())) {
15726 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQYrr, RC: &X86::VR256RegClass, Op0, Op1);
15727 }
15728 return 0;
15729}
15730
15731unsigned fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15732 switch (VT.SimpleTy) {
15733 case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op1);
15734 case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op1);
15735 case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op1);
15736 case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op1);
15737 case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op1);
15738 case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op1);
15739 case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op1);
15740 case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op1);
15741 default: return 0;
15742 }
15743}
15744
15745// FastEmit functions for X86ISD::PDEP.
15746
15747unsigned fastEmit_X86ISD_PDEP_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15748 if (RetVT.SimpleTy != MVT::i32)
15749 return 0;
15750 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
15751 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
15752 }
15753 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15754 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr, RC: &X86::GR32RegClass, Op0, Op1);
15755 }
15756 return 0;
15757}
15758
15759unsigned fastEmit_X86ISD_PDEP_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15760 if (RetVT.SimpleTy != MVT::i64)
15761 return 0;
15762 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
15763 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
15764 }
15765 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15766 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr, RC: &X86::GR64RegClass, Op0, Op1);
15767 }
15768 return 0;
15769}
15770
15771unsigned fastEmit_X86ISD_PDEP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15772 switch (VT.SimpleTy) {
15773 case MVT::i32: return fastEmit_X86ISD_PDEP_MVT_i32_rr(RetVT, Op0, Op1);
15774 case MVT::i64: return fastEmit_X86ISD_PDEP_MVT_i64_rr(RetVT, Op0, Op1);
15775 default: return 0;
15776 }
15777}
15778
15779// FastEmit functions for X86ISD::PEXT.
15780
15781unsigned fastEmit_X86ISD_PEXT_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15782 if (RetVT.SimpleTy != MVT::i32)
15783 return 0;
15784 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
15785 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
15786 }
15787 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15788 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr, RC: &X86::GR32RegClass, Op0, Op1);
15789 }
15790 return 0;
15791}
15792
15793unsigned fastEmit_X86ISD_PEXT_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15794 if (RetVT.SimpleTy != MVT::i64)
15795 return 0;
15796 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
15797 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
15798 }
15799 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15800 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr, RC: &X86::GR64RegClass, Op0, Op1);
15801 }
15802 return 0;
15803}
15804
15805unsigned fastEmit_X86ISD_PEXT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15806 switch (VT.SimpleTy) {
15807 case MVT::i32: return fastEmit_X86ISD_PEXT_MVT_i32_rr(RetVT, Op0, Op1);
15808 case MVT::i64: return fastEmit_X86ISD_PEXT_MVT_i64_rr(RetVT, Op0, Op1);
15809 default: return 0;
15810 }
15811}
15812
15813// FastEmit functions for X86ISD::PMULDQ.
15814
15815unsigned fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15816 if (RetVT.SimpleTy != MVT::v2i64)
15817 return 0;
15818 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15819 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15820 }
15821 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15822 return fastEmitInst_rr(MachineInstOpcode: X86::PMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
15823 }
15824 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15825 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
15826 }
15827 return 0;
15828}
15829
15830unsigned fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15831 if (RetVT.SimpleTy != MVT::v4i64)
15832 return 0;
15833 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15834 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15835 }
15836 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15837 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
15838 }
15839 return 0;
15840}
15841
15842unsigned fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15843 if (RetVT.SimpleTy != MVT::v8i64)
15844 return 0;
15845 if ((Subtarget->hasAVX512())) {
15846 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15847 }
15848 return 0;
15849}
15850
15851unsigned fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15852 switch (VT.SimpleTy) {
15853 case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
15854 case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
15855 case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
15856 default: return 0;
15857 }
15858}
15859
15860// FastEmit functions for X86ISD::PMULUDQ.
15861
15862unsigned fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15863 if (RetVT.SimpleTy != MVT::v2i64)
15864 return 0;
15865 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15866 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15867 }
15868 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15869 return fastEmitInst_rr(MachineInstOpcode: X86::PMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
15870 }
15871 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15872 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
15873 }
15874 return 0;
15875}
15876
15877unsigned fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15878 if (RetVT.SimpleTy != MVT::v4i64)
15879 return 0;
15880 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15881 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15882 }
15883 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15884 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
15885 }
15886 return 0;
15887}
15888
15889unsigned fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15890 if (RetVT.SimpleTy != MVT::v8i64)
15891 return 0;
15892 if ((Subtarget->hasAVX512())) {
15893 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15894 }
15895 return 0;
15896}
15897
15898unsigned fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15899 switch (VT.SimpleTy) {
15900 case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
15901 case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
15902 case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
15903 default: return 0;
15904 }
15905}
15906
15907// FastEmit functions for X86ISD::PSADBW.
15908
15909unsigned fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15910 if (RetVT.SimpleTy != MVT::v2i64)
15911 return 0;
15912 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15913 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15914 }
15915 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15916 return fastEmitInst_rr(MachineInstOpcode: X86::PSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
15917 }
15918 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15919 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
15920 }
15921 return 0;
15922}
15923
15924unsigned fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15925 if (RetVT.SimpleTy != MVT::v4i64)
15926 return 0;
15927 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15928 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15929 }
15930 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15931 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15932 }
15933 return 0;
15934}
15935
15936unsigned fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15937 if (RetVT.SimpleTy != MVT::v8i64)
15938 return 0;
15939 if ((Subtarget->hasBWI())) {
15940 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
15941 }
15942 return 0;
15943}
15944
15945unsigned fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15946 switch (VT.SimpleTy) {
15947 case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op1);
15948 case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op1);
15949 case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op1);
15950 default: return 0;
15951 }
15952}
15953
15954// FastEmit functions for X86ISD::PSHUFB.
15955
15956unsigned fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15957 if (RetVT.SimpleTy != MVT::v16i8)
15958 return 0;
15959 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15960 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15961 }
15962 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
15963 return fastEmitInst_rr(MachineInstOpcode: X86::PSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
15964 }
15965 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15966 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
15967 }
15968 return 0;
15969}
15970
15971unsigned fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15972 if (RetVT.SimpleTy != MVT::v32i8)
15973 return 0;
15974 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15975 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15976 }
15977 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15978 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBYrr, RC: &X86::VR256RegClass, Op0, Op1);
15979 }
15980 return 0;
15981}
15982
15983unsigned fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15984 if (RetVT.SimpleTy != MVT::v64i8)
15985 return 0;
15986 if ((Subtarget->hasBWI())) {
15987 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZrr, RC: &X86::VR512RegClass, Op0, Op1);
15988 }
15989 return 0;
15990}
15991
15992unsigned fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15993 switch (VT.SimpleTy) {
15994 case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op1);
15995 case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op1);
15996 case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op1);
15997 default: return 0;
15998 }
15999}
16000
16001// FastEmit functions for X86ISD::PTEST.
16002
16003unsigned fastEmit_X86ISD_PTEST_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16004 if (RetVT.SimpleTy != MVT::i32)
16005 return 0;
16006 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
16007 return fastEmitInst_rr(MachineInstOpcode: X86::PTESTrr, RC: &X86::VR128RegClass, Op0, Op1);
16008 }
16009 if ((Subtarget->hasAVX())) {
16010 return fastEmitInst_rr(MachineInstOpcode: X86::VPTESTrr, RC: &X86::VR128RegClass, Op0, Op1);
16011 }
16012 return 0;
16013}
16014
16015unsigned fastEmit_X86ISD_PTEST_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16016 if (RetVT.SimpleTy != MVT::i32)
16017 return 0;
16018 if ((Subtarget->hasAVX())) {
16019 return fastEmitInst_rr(MachineInstOpcode: X86::VPTESTYrr, RC: &X86::VR256RegClass, Op0, Op1);
16020 }
16021 return 0;
16022}
16023
16024unsigned fastEmit_X86ISD_PTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16025 switch (VT.SimpleTy) {
16026 case MVT::v2i64: return fastEmit_X86ISD_PTEST_MVT_v2i64_rr(RetVT, Op0, Op1);
16027 case MVT::v4i64: return fastEmit_X86ISD_PTEST_MVT_v4i64_rr(RetVT, Op0, Op1);
16028 default: return 0;
16029 }
16030}
16031
16032// FastEmit functions for X86ISD::RCP14S.
16033
16034unsigned fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16035 if (RetVT.SimpleTy != MVT::v8f16)
16036 return 0;
16037 if ((Subtarget->hasFP16())) {
16038 return fastEmitInst_rr(MachineInstOpcode: X86::VRCPSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16039 }
16040 return 0;
16041}
16042
16043unsigned fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16044 if (RetVT.SimpleTy != MVT::v4f32)
16045 return 0;
16046 if ((Subtarget->hasAVX512())) {
16047 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16048 }
16049 return 0;
16050}
16051
16052unsigned fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16053 if (RetVT.SimpleTy != MVT::v2f64)
16054 return 0;
16055 if ((Subtarget->hasAVX512())) {
16056 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16057 }
16058 return 0;
16059}
16060
16061unsigned fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16062 switch (VT.SimpleTy) {
16063 case MVT::v8f16: return fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(RetVT, Op0, Op1);
16064 case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op1);
16065 case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op1);
16066 default: return 0;
16067 }
16068}
16069
16070// FastEmit functions for X86ISD::RSQRT14S.
16071
16072unsigned fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16073 if (RetVT.SimpleTy != MVT::v8f16)
16074 return 0;
16075 if ((Subtarget->hasFP16())) {
16076 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRTSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16077 }
16078 return 0;
16079}
16080
16081unsigned fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16082 if (RetVT.SimpleTy != MVT::v4f32)
16083 return 0;
16084 if ((Subtarget->hasAVX512())) {
16085 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16086 }
16087 return 0;
16088}
16089
16090unsigned fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16091 if (RetVT.SimpleTy != MVT::v2f64)
16092 return 0;
16093 if ((Subtarget->hasAVX512())) {
16094 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16095 }
16096 return 0;
16097}
16098
16099unsigned fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16100 switch (VT.SimpleTy) {
16101 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(RetVT, Op0, Op1);
16102 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op1);
16103 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op1);
16104 default: return 0;
16105 }
16106}
16107
16108// FastEmit functions for X86ISD::SCALEF.
16109
16110unsigned fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16111 if (RetVT.SimpleTy != MVT::v8f16)
16112 return 0;
16113 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16114 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16115 }
16116 return 0;
16117}
16118
16119unsigned fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16120 if (RetVT.SimpleTy != MVT::v16f16)
16121 return 0;
16122 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16123 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16124 }
16125 return 0;
16126}
16127
16128unsigned fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16129 if (RetVT.SimpleTy != MVT::v32f16)
16130 return 0;
16131 if ((Subtarget->hasFP16())) {
16132 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16133 }
16134 return 0;
16135}
16136
16137unsigned fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16138 if (RetVT.SimpleTy != MVT::v4f32)
16139 return 0;
16140 if ((Subtarget->hasVLX())) {
16141 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16142 }
16143 return 0;
16144}
16145
16146unsigned fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16147 if (RetVT.SimpleTy != MVT::v8f32)
16148 return 0;
16149 if ((Subtarget->hasVLX())) {
16150 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16151 }
16152 return 0;
16153}
16154
16155unsigned fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16156 if (RetVT.SimpleTy != MVT::v16f32)
16157 return 0;
16158 if ((Subtarget->hasAVX512())) {
16159 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16160 }
16161 return 0;
16162}
16163
16164unsigned fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16165 if (RetVT.SimpleTy != MVT::v2f64)
16166 return 0;
16167 if ((Subtarget->hasVLX())) {
16168 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16169 }
16170 return 0;
16171}
16172
16173unsigned fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16174 if (RetVT.SimpleTy != MVT::v4f64)
16175 return 0;
16176 if ((Subtarget->hasVLX())) {
16177 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16178 }
16179 return 0;
16180}
16181
16182unsigned fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16183 if (RetVT.SimpleTy != MVT::v8f64)
16184 return 0;
16185 if ((Subtarget->hasAVX512())) {
16186 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16187 }
16188 return 0;
16189}
16190
16191unsigned fastEmit_X86ISD_SCALEF_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16192 switch (VT.SimpleTy) {
16193 case MVT::v8f16: return fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(RetVT, Op0, Op1);
16194 case MVT::v16f16: return fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(RetVT, Op0, Op1);
16195 case MVT::v32f16: return fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(RetVT, Op0, Op1);
16196 case MVT::v4f32: return fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(RetVT, Op0, Op1);
16197 case MVT::v8f32: return fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(RetVT, Op0, Op1);
16198 case MVT::v16f32: return fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(RetVT, Op0, Op1);
16199 case MVT::v2f64: return fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(RetVT, Op0, Op1);
16200 case MVT::v4f64: return fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(RetVT, Op0, Op1);
16201 case MVT::v8f64: return fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(RetVT, Op0, Op1);
16202 default: return 0;
16203 }
16204}
16205
16206// FastEmit functions for X86ISD::SCALEFS.
16207
16208unsigned fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16209 if (RetVT.SimpleTy != MVT::v8f16)
16210 return 0;
16211 if ((Subtarget->hasFP16())) {
16212 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16213 }
16214 return 0;
16215}
16216
16217unsigned fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16218 if (RetVT.SimpleTy != MVT::v4f32)
16219 return 0;
16220 if ((Subtarget->hasAVX512())) {
16221 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16222 }
16223 return 0;
16224}
16225
16226unsigned fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16227 if (RetVT.SimpleTy != MVT::v2f64)
16228 return 0;
16229 if ((Subtarget->hasAVX512())) {
16230 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16231 }
16232 return 0;
16233}
16234
16235unsigned fastEmit_X86ISD_SCALEFS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16236 switch (VT.SimpleTy) {
16237 case MVT::v8f16: return fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(RetVT, Op0, Op1);
16238 case MVT::v4f32: return fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(RetVT, Op0, Op1);
16239 case MVT::v2f64: return fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(RetVT, Op0, Op1);
16240 default: return 0;
16241 }
16242}
16243
16244// FastEmit functions for X86ISD::STRICT_FCMP.
16245
16246unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16247 if (RetVT.SimpleTy != MVT::i32)
16248 return 0;
16249 if ((Subtarget->hasFP16())) {
16250 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16251 }
16252 return 0;
16253}
16254
16255unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16256 if (RetVT.SimpleTy != MVT::i32)
16257 return 0;
16258 if ((Subtarget->hasAVX512())) {
16259 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16260 }
16261 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16262 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
16263 }
16264 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16265 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
16266 }
16267 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
16268 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
16269 }
16270 return 0;
16271}
16272
16273unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16274 if (RetVT.SimpleTy != MVT::i32)
16275 return 0;
16276 if ((Subtarget->hasAVX512())) {
16277 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16278 }
16279 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16280 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
16281 }
16282 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16283 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
16284 }
16285 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
16286 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
16287 }
16288 return 0;
16289}
16290
16291unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16292 if (RetVT.SimpleTy != MVT::i32)
16293 return 0;
16294 if ((Subtarget->canUseCMOV())) {
16295 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
16296 }
16297 return 0;
16298}
16299
16300unsigned fastEmit_X86ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16301 switch (VT.SimpleTy) {
16302 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
16303 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
16304 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
16305 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
16306 default: return 0;
16307 }
16308}
16309
16310// FastEmit functions for X86ISD::STRICT_FCMPS.
16311
16312unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16313 if (RetVT.SimpleTy != MVT::i32)
16314 return 0;
16315 if ((Subtarget->hasFP16())) {
16316 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16317 }
16318 return 0;
16319}
16320
16321unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16322 if (RetVT.SimpleTy != MVT::i32)
16323 return 0;
16324 if ((Subtarget->hasAVX512())) {
16325 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16326 }
16327 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16328 return fastEmitInst_rr(MachineInstOpcode: X86::COMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
16329 }
16330 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16331 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
16332 }
16333 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
16334 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
16335 }
16336 return 0;
16337}
16338
16339unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16340 if (RetVT.SimpleTy != MVT::i32)
16341 return 0;
16342 if ((Subtarget->hasAVX512())) {
16343 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16344 }
16345 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16346 return fastEmitInst_rr(MachineInstOpcode: X86::COMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
16347 }
16348 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16349 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
16350 }
16351 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
16352 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
16353 }
16354 return 0;
16355}
16356
16357unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16358 if (RetVT.SimpleTy != MVT::i32)
16359 return 0;
16360 if ((Subtarget->canUseCMOV())) {
16361 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
16362 }
16363 return 0;
16364}
16365
16366unsigned fastEmit_X86ISD_STRICT_FCMPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16367 switch (VT.SimpleTy) {
16368 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(RetVT, Op0, Op1);
16369 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(RetVT, Op0, Op1);
16370 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(RetVT, Op0, Op1);
16371 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(RetVT, Op0, Op1);
16372 default: return 0;
16373 }
16374}
16375
16376// FastEmit functions for X86ISD::STRICT_FP80_ADD.
16377
16378unsigned fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16379 if (RetVT.SimpleTy != MVT::f80)
16380 return 0;
16381 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
16382}
16383
16384unsigned fastEmit_X86ISD_STRICT_FP80_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16385 switch (VT.SimpleTy) {
16386 case MVT::f80: return fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
16387 default: return 0;
16388 }
16389}
16390
16391// FastEmit functions for X86ISD::TESTP.
16392
16393unsigned fastEmit_X86ISD_TESTP_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16394 if (RetVT.SimpleTy != MVT::i32)
16395 return 0;
16396 if ((Subtarget->hasAVX())) {
16397 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16398 }
16399 return 0;
16400}
16401
16402unsigned fastEmit_X86ISD_TESTP_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16403 if (RetVT.SimpleTy != MVT::i32)
16404 return 0;
16405 if ((Subtarget->hasAVX())) {
16406 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16407 }
16408 return 0;
16409}
16410
16411unsigned fastEmit_X86ISD_TESTP_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16412 if (RetVT.SimpleTy != MVT::i32)
16413 return 0;
16414 if ((Subtarget->hasAVX())) {
16415 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16416 }
16417 return 0;
16418}
16419
16420unsigned fastEmit_X86ISD_TESTP_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16421 if (RetVT.SimpleTy != MVT::i32)
16422 return 0;
16423 if ((Subtarget->hasAVX())) {
16424 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16425 }
16426 return 0;
16427}
16428
16429unsigned fastEmit_X86ISD_TESTP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16430 switch (VT.SimpleTy) {
16431 case MVT::v4f32: return fastEmit_X86ISD_TESTP_MVT_v4f32_rr(RetVT, Op0, Op1);
16432 case MVT::v8f32: return fastEmit_X86ISD_TESTP_MVT_v8f32_rr(RetVT, Op0, Op1);
16433 case MVT::v2f64: return fastEmit_X86ISD_TESTP_MVT_v2f64_rr(RetVT, Op0, Op1);
16434 case MVT::v4f64: return fastEmit_X86ISD_TESTP_MVT_v4f64_rr(RetVT, Op0, Op1);
16435 default: return 0;
16436 }
16437}
16438
16439// FastEmit functions for X86ISD::UCOMI.
16440
16441unsigned fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16442 if (RetVT.SimpleTy != MVT::i32)
16443 return 0;
16444 if ((Subtarget->hasFP16())) {
16445 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16446 }
16447 return 0;
16448}
16449
16450unsigned fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16451 if (RetVT.SimpleTy != MVT::i32)
16452 return 0;
16453 if ((Subtarget->hasAVX512())) {
16454 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16455 }
16456 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16457 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16458 }
16459 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16460 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16461 }
16462 return 0;
16463}
16464
16465unsigned fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16466 if (RetVT.SimpleTy != MVT::i32)
16467 return 0;
16468 if ((Subtarget->hasAVX512())) {
16469 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16470 }
16471 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16472 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16473 }
16474 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16475 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16476 }
16477 return 0;
16478}
16479
16480unsigned fastEmit_X86ISD_UCOMI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16481 switch (VT.SimpleTy) {
16482 case MVT::v8f16: return fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(RetVT, Op0, Op1);
16483 case MVT::v4f32: return fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(RetVT, Op0, Op1);
16484 case MVT::v2f64: return fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(RetVT, Op0, Op1);
16485 default: return 0;
16486 }
16487}
16488
16489// FastEmit functions for X86ISD::UNPCKH.
16490
16491unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16492 if (RetVT.SimpleTy != MVT::v16i8)
16493 return 0;
16494 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16495 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16496 }
16497 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16498 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
16499 }
16500 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16501 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
16502 }
16503 return 0;
16504}
16505
16506unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16507 if (RetVT.SimpleTy != MVT::v32i8)
16508 return 0;
16509 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16510 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16511 }
16512 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16513 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16514 }
16515 return 0;
16516}
16517
16518unsigned fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16519 if (RetVT.SimpleTy != MVT::v64i8)
16520 return 0;
16521 if ((Subtarget->hasBWI())) {
16522 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
16523 }
16524 return 0;
16525}
16526
16527unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16528 if (RetVT.SimpleTy != MVT::v8i16)
16529 return 0;
16530 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16531 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16532 }
16533 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16534 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
16535 }
16536 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16537 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
16538 }
16539 return 0;
16540}
16541
16542unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16543 if (RetVT.SimpleTy != MVT::v16i16)
16544 return 0;
16545 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16546 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16547 }
16548 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16549 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16550 }
16551 return 0;
16552}
16553
16554unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16555 if (RetVT.SimpleTy != MVT::v32i16)
16556 return 0;
16557 if ((Subtarget->hasBWI())) {
16558 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16559 }
16560 return 0;
16561}
16562
16563unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16564 if (RetVT.SimpleTy != MVT::v4i32)
16565 return 0;
16566 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16567 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16568 }
16569 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16570 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
16571 }
16572 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16573 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
16574 }
16575 return 0;
16576}
16577
16578unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16579 if (RetVT.SimpleTy != MVT::v8i32)
16580 return 0;
16581 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
16582 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16583 }
16584 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16585 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16586 }
16587 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
16588 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
16589 }
16590 return 0;
16591}
16592
16593unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16594 if (RetVT.SimpleTy != MVT::v16i32)
16595 return 0;
16596 if ((Subtarget->hasAVX512())) {
16597 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
16598 }
16599 return 0;
16600}
16601
16602unsigned fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16603 if (RetVT.SimpleTy != MVT::v2i64)
16604 return 0;
16605 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16606 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16607 }
16608 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16609 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
16610 }
16611 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16612 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
16613 }
16614 return 0;
16615}
16616
16617unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16618 if (RetVT.SimpleTy != MVT::v4i64)
16619 return 0;
16620 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
16621 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16622 }
16623 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16624 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16625 }
16626 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
16627 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
16628 }
16629 return 0;
16630}
16631
16632unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16633 if (RetVT.SimpleTy != MVT::v8i64)
16634 return 0;
16635 if ((Subtarget->hasAVX512())) {
16636 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
16637 }
16638 return 0;
16639}
16640
16641unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16642 if (RetVT.SimpleTy != MVT::v4f32)
16643 return 0;
16644 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16645 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16646 }
16647 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16648 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16649 }
16650 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16651 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16652 }
16653 return 0;
16654}
16655
16656unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16657 if (RetVT.SimpleTy != MVT::v8f32)
16658 return 0;
16659 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16660 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16661 }
16662 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16663 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16664 }
16665 return 0;
16666}
16667
16668unsigned fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16669 if (RetVT.SimpleTy != MVT::v16f32)
16670 return 0;
16671 if ((Subtarget->hasAVX512())) {
16672 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16673 }
16674 return 0;
16675}
16676
16677unsigned fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16678 if (RetVT.SimpleTy != MVT::v2f64)
16679 return 0;
16680 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16681 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16682 }
16683 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16684 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16685 }
16686 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16687 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16688 }
16689 return 0;
16690}
16691
16692unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16693 if (RetVT.SimpleTy != MVT::v4f64)
16694 return 0;
16695 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16696 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16697 }
16698 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16699 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16700 }
16701 return 0;
16702}
16703
16704unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16705 if (RetVT.SimpleTy != MVT::v8f64)
16706 return 0;
16707 if ((Subtarget->hasAVX512())) {
16708 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16709 }
16710 return 0;
16711}
16712
16713unsigned fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16714 switch (VT.SimpleTy) {
16715 case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op1);
16716 case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op1);
16717 case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op1);
16718 case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op1);
16719 case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op1);
16720 case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op1);
16721 case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op1);
16722 case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op1);
16723 case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op1);
16724 case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op1);
16725 case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op1);
16726 case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op1);
16727 case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op1);
16728 case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op1);
16729 case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op1);
16730 case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op1);
16731 case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op1);
16732 case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op1);
16733 default: return 0;
16734 }
16735}
16736
16737// FastEmit functions for X86ISD::UNPCKL.
16738
16739unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16740 if (RetVT.SimpleTy != MVT::v16i8)
16741 return 0;
16742 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16743 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16744 }
16745 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16746 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
16747 }
16748 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16749 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
16750 }
16751 return 0;
16752}
16753
16754unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16755 if (RetVT.SimpleTy != MVT::v32i8)
16756 return 0;
16757 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16758 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16759 }
16760 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16761 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16762 }
16763 return 0;
16764}
16765
16766unsigned fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16767 if (RetVT.SimpleTy != MVT::v64i8)
16768 return 0;
16769 if ((Subtarget->hasBWI())) {
16770 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
16771 }
16772 return 0;
16773}
16774
16775unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16776 if (RetVT.SimpleTy != MVT::v8i16)
16777 return 0;
16778 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16779 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16780 }
16781 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16782 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
16783 }
16784 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16785 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
16786 }
16787 return 0;
16788}
16789
16790unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16791 if (RetVT.SimpleTy != MVT::v16i16)
16792 return 0;
16793 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16794 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16795 }
16796 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16797 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16798 }
16799 return 0;
16800}
16801
16802unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16803 if (RetVT.SimpleTy != MVT::v32i16)
16804 return 0;
16805 if ((Subtarget->hasBWI())) {
16806 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16807 }
16808 return 0;
16809}
16810
16811unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16812 if (RetVT.SimpleTy != MVT::v4i32)
16813 return 0;
16814 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16815 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16816 }
16817 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16818 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
16819 }
16820 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16821 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
16822 }
16823 return 0;
16824}
16825
16826unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16827 if (RetVT.SimpleTy != MVT::v8i32)
16828 return 0;
16829 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
16830 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16831 }
16832 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16833 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16834 }
16835 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
16836 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
16837 }
16838 return 0;
16839}
16840
16841unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16842 if (RetVT.SimpleTy != MVT::v16i32)
16843 return 0;
16844 if ((Subtarget->hasAVX512())) {
16845 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
16846 }
16847 return 0;
16848}
16849
16850unsigned fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16851 if (RetVT.SimpleTy != MVT::v2i64)
16852 return 0;
16853 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16854 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16855 }
16856 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16857 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
16858 }
16859 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16860 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
16861 }
16862 return 0;
16863}
16864
16865unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16866 if (RetVT.SimpleTy != MVT::v4i64)
16867 return 0;
16868 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
16869 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16870 }
16871 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16872 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16873 }
16874 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
16875 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
16876 }
16877 return 0;
16878}
16879
16880unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16881 if (RetVT.SimpleTy != MVT::v8i64)
16882 return 0;
16883 if ((Subtarget->hasAVX512())) {
16884 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
16885 }
16886 return 0;
16887}
16888
16889unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16890 if (RetVT.SimpleTy != MVT::v4f32)
16891 return 0;
16892 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16893 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16894 }
16895 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16896 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16897 }
16898 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16899 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16900 }
16901 return 0;
16902}
16903
16904unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16905 if (RetVT.SimpleTy != MVT::v8f32)
16906 return 0;
16907 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16908 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16909 }
16910 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16911 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16912 }
16913 return 0;
16914}
16915
16916unsigned fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16917 if (RetVT.SimpleTy != MVT::v16f32)
16918 return 0;
16919 if ((Subtarget->hasAVX512())) {
16920 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16921 }
16922 return 0;
16923}
16924
16925unsigned fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16926 if (RetVT.SimpleTy != MVT::v2f64)
16927 return 0;
16928 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16929 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16930 }
16931 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16932 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16933 }
16934 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16935 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16936 }
16937 return 0;
16938}
16939
16940unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16941 if (RetVT.SimpleTy != MVT::v4f64)
16942 return 0;
16943 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16944 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16945 }
16946 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16947 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16948 }
16949 return 0;
16950}
16951
16952unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16953 if (RetVT.SimpleTy != MVT::v8f64)
16954 return 0;
16955 if ((Subtarget->hasAVX512())) {
16956 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16957 }
16958 return 0;
16959}
16960
16961unsigned fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16962 switch (VT.SimpleTy) {
16963 case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op1);
16964 case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op1);
16965 case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op1);
16966 case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op1);
16967 case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op1);
16968 case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op1);
16969 case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op1);
16970 case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op1);
16971 case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op1);
16972 case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op1);
16973 case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op1);
16974 case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op1);
16975 case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op1);
16976 case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op1);
16977 case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op1);
16978 case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op1);
16979 case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op1);
16980 case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op1);
16981 default: return 0;
16982 }
16983}
16984
16985// FastEmit functions for X86ISD::VFCMULC.
16986
16987unsigned fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16988 if (RetVT.SimpleTy != MVT::v4f32)
16989 return 0;
16990 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16991 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16992 }
16993 return 0;
16994}
16995
16996unsigned fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16997 if (RetVT.SimpleTy != MVT::v8f32)
16998 return 0;
16999 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17000 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17001 }
17002 return 0;
17003}
17004
17005unsigned fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17006 if (RetVT.SimpleTy != MVT::v16f32)
17007 return 0;
17008 if ((Subtarget->hasFP16())) {
17009 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
17010 }
17011 return 0;
17012}
17013
17014unsigned fastEmit_X86ISD_VFCMULC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17015 switch (VT.SimpleTy) {
17016 case MVT::v4f32: return fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
17017 case MVT::v8f32: return fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
17018 case MVT::v16f32: return fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
17019 default: return 0;
17020 }
17021}
17022
17023// FastEmit functions for X86ISD::VFCMULCSH.
17024
17025unsigned fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17026 if (RetVT.SimpleTy != MVT::v4f32)
17027 return 0;
17028 if ((Subtarget->hasFP16())) {
17029 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17030 }
17031 return 0;
17032}
17033
17034unsigned fastEmit_X86ISD_VFCMULCSH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17035 switch (VT.SimpleTy) {
17036 case MVT::v4f32: return fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
17037 default: return 0;
17038 }
17039}
17040
17041// FastEmit functions for X86ISD::VFMULC.
17042
17043unsigned fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17044 if (RetVT.SimpleTy != MVT::v4f32)
17045 return 0;
17046 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17047 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17048 }
17049 return 0;
17050}
17051
17052unsigned fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17053 if (RetVT.SimpleTy != MVT::v8f32)
17054 return 0;
17055 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17056 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17057 }
17058 return 0;
17059}
17060
17061unsigned fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17062 if (RetVT.SimpleTy != MVT::v16f32)
17063 return 0;
17064 if ((Subtarget->hasFP16())) {
17065 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
17066 }
17067 return 0;
17068}
17069
17070unsigned fastEmit_X86ISD_VFMULC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17071 switch (VT.SimpleTy) {
17072 case MVT::v4f32: return fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
17073 case MVT::v8f32: return fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
17074 case MVT::v16f32: return fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
17075 default: return 0;
17076 }
17077}
17078
17079// FastEmit functions for X86ISD::VFMULCSH.
17080
17081unsigned fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17082 if (RetVT.SimpleTy != MVT::v4f32)
17083 return 0;
17084 if ((Subtarget->hasFP16())) {
17085 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17086 }
17087 return 0;
17088}
17089
17090unsigned fastEmit_X86ISD_VFMULCSH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17091 switch (VT.SimpleTy) {
17092 case MVT::v4f32: return fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
17093 default: return 0;
17094 }
17095}
17096
17097// FastEmit functions for X86ISD::VP2INTERSECT.
17098
17099unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17100 if (RetVT.SimpleTy != MVT::Untyped)
17101 return 0;
17102 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
17103 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ128rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
17104 }
17105 return 0;
17106}
17107
17108unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17109 if (RetVT.SimpleTy != MVT::Untyped)
17110 return 0;
17111 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
17112 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ256rr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
17113 }
17114 return 0;
17115}
17116
17117unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17118 if (RetVT.SimpleTy != MVT::Untyped)
17119 return 0;
17120 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
17121 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZrr, RC: &X86::VK16PAIRRegClass, Op0, Op1);
17122 }
17123 return 0;
17124}
17125
17126unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17127 if (RetVT.SimpleTy != MVT::Untyped)
17128 return 0;
17129 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
17130 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ128rr, RC: &X86::VK2PAIRRegClass, Op0, Op1);
17131 }
17132 return 0;
17133}
17134
17135unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17136 if (RetVT.SimpleTy != MVT::Untyped)
17137 return 0;
17138 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
17139 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ256rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
17140 }
17141 return 0;
17142}
17143
17144unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17145 if (RetVT.SimpleTy != MVT::Untyped)
17146 return 0;
17147 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
17148 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZrr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
17149 }
17150 return 0;
17151}
17152
17153unsigned fastEmit_X86ISD_VP2INTERSECT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17154 switch (VT.SimpleTy) {
17155 case MVT::v4i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(RetVT, Op0, Op1);
17156 case MVT::v8i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(RetVT, Op0, Op1);
17157 case MVT::v16i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(RetVT, Op0, Op1);
17158 case MVT::v2i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(RetVT, Op0, Op1);
17159 case MVT::v4i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(RetVT, Op0, Op1);
17160 case MVT::v8i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(RetVT, Op0, Op1);
17161 default: return 0;
17162 }
17163}
17164
17165// FastEmit functions for X86ISD::VPERMV.
17166
17167unsigned fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17168 if (RetVT.SimpleTy != MVT::v16i8)
17169 return 0;
17170 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17171 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17172 }
17173 return 0;
17174}
17175
17176unsigned fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17177 if (RetVT.SimpleTy != MVT::v32i8)
17178 return 0;
17179 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17180 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17181 }
17182 return 0;
17183}
17184
17185unsigned fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17186 if (RetVT.SimpleTy != MVT::v64i8)
17187 return 0;
17188 if ((Subtarget->hasVBMI())) {
17189 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17190 }
17191 return 0;
17192}
17193
17194unsigned fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17195 if (RetVT.SimpleTy != MVT::v8i16)
17196 return 0;
17197 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17198 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17199 }
17200 return 0;
17201}
17202
17203unsigned fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17204 if (RetVT.SimpleTy != MVT::v16i16)
17205 return 0;
17206 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17207 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17208 }
17209 return 0;
17210}
17211
17212unsigned fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17213 if (RetVT.SimpleTy != MVT::v32i16)
17214 return 0;
17215 if ((Subtarget->hasBWI())) {
17216 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17217 }
17218 return 0;
17219}
17220
17221unsigned fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17222 if (RetVT.SimpleTy != MVT::v8i32)
17223 return 0;
17224 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17225 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17226 }
17227 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17228 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17229 }
17230 return 0;
17231}
17232
17233unsigned fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17234 if (RetVT.SimpleTy != MVT::v16i32)
17235 return 0;
17236 if ((Subtarget->hasAVX512())) {
17237 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZrr, RC: &X86::VR512RegClass, Op0, Op1);
17238 }
17239 return 0;
17240}
17241
17242unsigned fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17243 if (RetVT.SimpleTy != MVT::v4i64)
17244 return 0;
17245 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17246 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17247 }
17248 return 0;
17249}
17250
17251unsigned fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17252 if (RetVT.SimpleTy != MVT::v8i64)
17253 return 0;
17254 if ((Subtarget->hasAVX512())) {
17255 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17256 }
17257 return 0;
17258}
17259
17260unsigned fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17261 switch (VT.SimpleTy) {
17262 case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op1);
17263 case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op1);
17264 case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op1);
17265 case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op1);
17266 case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op1);
17267 case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op1);
17268 case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op1);
17269 case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op1);
17270 case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op1);
17271 case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op1);
17272 default: return 0;
17273 }
17274}
17275
17276// FastEmit functions for X86ISD::VPMADDUBSW.
17277
17278unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17279 if (RetVT.SimpleTy != MVT::v8i16)
17280 return 0;
17281 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17282 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17283 }
17284 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17285 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17286 }
17287 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17288 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17289 }
17290 return 0;
17291}
17292
17293unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17294 if (RetVT.SimpleTy != MVT::v16i16)
17295 return 0;
17296 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17297 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17298 }
17299 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17300 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17301 }
17302 return 0;
17303}
17304
17305unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17306 if (RetVT.SimpleTy != MVT::v32i16)
17307 return 0;
17308 if ((Subtarget->hasBWI())) {
17309 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17310 }
17311 return 0;
17312}
17313
17314unsigned fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17315 switch (VT.SimpleTy) {
17316 case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op1);
17317 case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op1);
17318 case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op1);
17319 default: return 0;
17320 }
17321}
17322
17323// FastEmit functions for X86ISD::VPMADDWD.
17324
17325unsigned fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17326 if (RetVT.SimpleTy != MVT::v4i32)
17327 return 0;
17328 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17329 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17330 }
17331 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17332 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
17333 }
17334 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17335 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
17336 }
17337 return 0;
17338}
17339
17340unsigned fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17341 if (RetVT.SimpleTy != MVT::v8i32)
17342 return 0;
17343 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17344 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17345 }
17346 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17347 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17348 }
17349 return 0;
17350}
17351
17352unsigned fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17353 if (RetVT.SimpleTy != MVT::v16i32)
17354 return 0;
17355 if ((Subtarget->hasBWI())) {
17356 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
17357 }
17358 return 0;
17359}
17360
17361unsigned fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17362 switch (VT.SimpleTy) {
17363 case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op1);
17364 case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op1);
17365 case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op1);
17366 default: return 0;
17367 }
17368}
17369
17370// FastEmit functions for X86ISD::VPSHA.
17371
17372unsigned fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17373 if (RetVT.SimpleTy != MVT::v16i8)
17374 return 0;
17375 if ((Subtarget->hasXOP())) {
17376 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHABrr, RC: &X86::VR128RegClass, Op0, Op1);
17377 }
17378 return 0;
17379}
17380
17381unsigned fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17382 if (RetVT.SimpleTy != MVT::v8i16)
17383 return 0;
17384 if ((Subtarget->hasXOP())) {
17385 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAWrr, RC: &X86::VR128RegClass, Op0, Op1);
17386 }
17387 return 0;
17388}
17389
17390unsigned fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17391 if (RetVT.SimpleTy != MVT::v4i32)
17392 return 0;
17393 if ((Subtarget->hasXOP())) {
17394 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHADrr, RC: &X86::VR128RegClass, Op0, Op1);
17395 }
17396 return 0;
17397}
17398
17399unsigned fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17400 if (RetVT.SimpleTy != MVT::v2i64)
17401 return 0;
17402 if ((Subtarget->hasXOP())) {
17403 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAQrr, RC: &X86::VR128RegClass, Op0, Op1);
17404 }
17405 return 0;
17406}
17407
17408unsigned fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17409 switch (VT.SimpleTy) {
17410 case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op1);
17411 case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op1);
17412 case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op1);
17413 case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op1);
17414 default: return 0;
17415 }
17416}
17417
17418// FastEmit functions for X86ISD::VPSHL.
17419
17420unsigned fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17421 if (RetVT.SimpleTy != MVT::v16i8)
17422 return 0;
17423 if ((Subtarget->hasXOP())) {
17424 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLBrr, RC: &X86::VR128RegClass, Op0, Op1);
17425 }
17426 return 0;
17427}
17428
17429unsigned fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17430 if (RetVT.SimpleTy != MVT::v8i16)
17431 return 0;
17432 if ((Subtarget->hasXOP())) {
17433 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLWrr, RC: &X86::VR128RegClass, Op0, Op1);
17434 }
17435 return 0;
17436}
17437
17438unsigned fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17439 if (RetVT.SimpleTy != MVT::v4i32)
17440 return 0;
17441 if ((Subtarget->hasXOP())) {
17442 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLDrr, RC: &X86::VR128RegClass, Op0, Op1);
17443 }
17444 return 0;
17445}
17446
17447unsigned fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17448 if (RetVT.SimpleTy != MVT::v2i64)
17449 return 0;
17450 if ((Subtarget->hasXOP())) {
17451 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLQrr, RC: &X86::VR128RegClass, Op0, Op1);
17452 }
17453 return 0;
17454}
17455
17456unsigned fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17457 switch (VT.SimpleTy) {
17458 case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op1);
17459 case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
17460 case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
17461 case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
17462 default: return 0;
17463 }
17464}
17465
17466// FastEmit functions for X86ISD::VPSHUFBITQMB.
17467
17468unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17469 if (RetVT.SimpleTy != MVT::v16i1)
17470 return 0;
17471 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
17472 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ128rr, RC: &X86::VK16RegClass, Op0, Op1);
17473 }
17474 return 0;
17475}
17476
17477unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17478 if (RetVT.SimpleTy != MVT::v32i1)
17479 return 0;
17480 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
17481 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ256rr, RC: &X86::VK32RegClass, Op0, Op1);
17482 }
17483 return 0;
17484}
17485
17486unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17487 if (RetVT.SimpleTy != MVT::v64i1)
17488 return 0;
17489 if ((Subtarget->hasBITALG())) {
17490 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZrr, RC: &X86::VK64RegClass, Op0, Op1);
17491 }
17492 return 0;
17493}
17494
17495unsigned fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17496 switch (VT.SimpleTy) {
17497 case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op1);
17498 case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op1);
17499 case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op1);
17500 default: return 0;
17501 }
17502}
17503
17504// FastEmit functions for X86ISD::VSHL.
17505
17506unsigned fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17507 if (RetVT.SimpleTy != MVT::v8i16)
17508 return 0;
17509 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17510 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17511 }
17512 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17513 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
17514 }
17515 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17516 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
17517 }
17518 return 0;
17519}
17520
17521unsigned fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17522 if (RetVT.SimpleTy != MVT::v4i32)
17523 return 0;
17524 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17525 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17526 }
17527 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17528 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
17529 }
17530 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17531 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
17532 }
17533 return 0;
17534}
17535
17536unsigned fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17537 if (RetVT.SimpleTy != MVT::v2i64)
17538 return 0;
17539 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17540 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17541 }
17542 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17543 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
17544 }
17545 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17546 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
17547 }
17548 return 0;
17549}
17550
17551unsigned fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17552 switch (VT.SimpleTy) {
17553 case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
17554 case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
17555 case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
17556 default: return 0;
17557 }
17558}
17559
17560// FastEmit functions for X86ISD::VSHLV.
17561
17562unsigned fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17563 if (RetVT.SimpleTy != MVT::v8i16)
17564 return 0;
17565 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17566 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17567 }
17568 return 0;
17569}
17570
17571unsigned fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17572 if (RetVT.SimpleTy != MVT::v16i16)
17573 return 0;
17574 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17575 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17576 }
17577 return 0;
17578}
17579
17580unsigned fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17581 if (RetVT.SimpleTy != MVT::v32i16)
17582 return 0;
17583 if ((Subtarget->hasBWI())) {
17584 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17585 }
17586 return 0;
17587}
17588
17589unsigned fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17590 if (RetVT.SimpleTy != MVT::v4i32)
17591 return 0;
17592 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17593 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17594 }
17595 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17596 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
17597 }
17598 return 0;
17599}
17600
17601unsigned fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17602 if (RetVT.SimpleTy != MVT::v8i32)
17603 return 0;
17604 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17605 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17606 }
17607 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17608 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17609 }
17610 return 0;
17611}
17612
17613unsigned fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17614 if (RetVT.SimpleTy != MVT::v16i32)
17615 return 0;
17616 if ((Subtarget->hasAVX512())) {
17617 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
17618 }
17619 return 0;
17620}
17621
17622unsigned fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17623 if (RetVT.SimpleTy != MVT::v2i64)
17624 return 0;
17625 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17626 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17627 }
17628 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17629 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
17630 }
17631 return 0;
17632}
17633
17634unsigned fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17635 if (RetVT.SimpleTy != MVT::v4i64)
17636 return 0;
17637 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17638 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17639 }
17640 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17641 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17642 }
17643 return 0;
17644}
17645
17646unsigned fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17647 if (RetVT.SimpleTy != MVT::v8i64)
17648 return 0;
17649 if ((Subtarget->hasAVX512())) {
17650 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17651 }
17652 return 0;
17653}
17654
17655unsigned fastEmit_X86ISD_VSHLV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17656 switch (VT.SimpleTy) {
17657 case MVT::v8i16: return fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(RetVT, Op0, Op1);
17658 case MVT::v16i16: return fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(RetVT, Op0, Op1);
17659 case MVT::v32i16: return fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(RetVT, Op0, Op1);
17660 case MVT::v4i32: return fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(RetVT, Op0, Op1);
17661 case MVT::v8i32: return fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(RetVT, Op0, Op1);
17662 case MVT::v16i32: return fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(RetVT, Op0, Op1);
17663 case MVT::v2i64: return fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(RetVT, Op0, Op1);
17664 case MVT::v4i64: return fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(RetVT, Op0, Op1);
17665 case MVT::v8i64: return fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(RetVT, Op0, Op1);
17666 default: return 0;
17667 }
17668}
17669
17670// FastEmit functions for X86ISD::VSRA.
17671
17672unsigned fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17673 if (RetVT.SimpleTy != MVT::v8i16)
17674 return 0;
17675 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17676 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17677 }
17678 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17679 return fastEmitInst_rr(MachineInstOpcode: X86::PSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
17680 }
17681 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17682 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
17683 }
17684 return 0;
17685}
17686
17687unsigned fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17688 if (RetVT.SimpleTy != MVT::v4i32)
17689 return 0;
17690 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17691 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17692 }
17693 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17694 return fastEmitInst_rr(MachineInstOpcode: X86::PSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
17695 }
17696 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17697 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
17698 }
17699 return 0;
17700}
17701
17702unsigned fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17703 if (RetVT.SimpleTy != MVT::v2i64)
17704 return 0;
17705 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17706 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17707 }
17708 return 0;
17709}
17710
17711unsigned fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17712 switch (VT.SimpleTy) {
17713 case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op1);
17714 case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op1);
17715 case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op1);
17716 default: return 0;
17717 }
17718}
17719
17720// FastEmit functions for X86ISD::VSRAV.
17721
17722unsigned fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17723 if (RetVT.SimpleTy != MVT::v8i16)
17724 return 0;
17725 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17726 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17727 }
17728 return 0;
17729}
17730
17731unsigned fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17732 if (RetVT.SimpleTy != MVT::v16i16)
17733 return 0;
17734 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17735 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17736 }
17737 return 0;
17738}
17739
17740unsigned fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17741 if (RetVT.SimpleTy != MVT::v32i16)
17742 return 0;
17743 if ((Subtarget->hasBWI())) {
17744 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17745 }
17746 return 0;
17747}
17748
17749unsigned fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17750 if (RetVT.SimpleTy != MVT::v4i32)
17751 return 0;
17752 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17753 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17754 }
17755 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17756 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDrr, RC: &X86::VR128RegClass, Op0, Op1);
17757 }
17758 return 0;
17759}
17760
17761unsigned fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17762 if (RetVT.SimpleTy != MVT::v8i32)
17763 return 0;
17764 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17765 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17766 }
17767 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17768 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17769 }
17770 return 0;
17771}
17772
17773unsigned fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17774 if (RetVT.SimpleTy != MVT::v16i32)
17775 return 0;
17776 if ((Subtarget->hasAVX512())) {
17777 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
17778 }
17779 return 0;
17780}
17781
17782unsigned fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17783 if (RetVT.SimpleTy != MVT::v2i64)
17784 return 0;
17785 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17786 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17787 }
17788 return 0;
17789}
17790
17791unsigned fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17792 if (RetVT.SimpleTy != MVT::v4i64)
17793 return 0;
17794 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17795 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17796 }
17797 return 0;
17798}
17799
17800unsigned fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17801 if (RetVT.SimpleTy != MVT::v8i64)
17802 return 0;
17803 if ((Subtarget->hasAVX512())) {
17804 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17805 }
17806 return 0;
17807}
17808
17809unsigned fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17810 switch (VT.SimpleTy) {
17811 case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op1);
17812 case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op1);
17813 case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op1);
17814 case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op1);
17815 case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op1);
17816 case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op1);
17817 case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op1);
17818 case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op1);
17819 case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op1);
17820 default: return 0;
17821 }
17822}
17823
17824// FastEmit functions for X86ISD::VSRL.
17825
17826unsigned fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17827 if (RetVT.SimpleTy != MVT::v8i16)
17828 return 0;
17829 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17830 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17831 }
17832 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17833 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
17834 }
17835 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17836 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
17837 }
17838 return 0;
17839}
17840
17841unsigned fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17842 if (RetVT.SimpleTy != MVT::v4i32)
17843 return 0;
17844 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17845 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17846 }
17847 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17848 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
17849 }
17850 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17851 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
17852 }
17853 return 0;
17854}
17855
17856unsigned fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17857 if (RetVT.SimpleTy != MVT::v2i64)
17858 return 0;
17859 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17860 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17861 }
17862 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17863 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
17864 }
17865 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17866 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
17867 }
17868 return 0;
17869}
17870
17871unsigned fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17872 switch (VT.SimpleTy) {
17873 case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op1);
17874 case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op1);
17875 case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op1);
17876 default: return 0;
17877 }
17878}
17879
17880// FastEmit functions for X86ISD::VSRLV.
17881
17882unsigned fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17883 if (RetVT.SimpleTy != MVT::v8i16)
17884 return 0;
17885 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17886 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17887 }
17888 return 0;
17889}
17890
17891unsigned fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17892 if (RetVT.SimpleTy != MVT::v16i16)
17893 return 0;
17894 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17895 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17896 }
17897 return 0;
17898}
17899
17900unsigned fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17901 if (RetVT.SimpleTy != MVT::v32i16)
17902 return 0;
17903 if ((Subtarget->hasBWI())) {
17904 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17905 }
17906 return 0;
17907}
17908
17909unsigned fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17910 if (RetVT.SimpleTy != MVT::v4i32)
17911 return 0;
17912 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17913 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17914 }
17915 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17916 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
17917 }
17918 return 0;
17919}
17920
17921unsigned fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17922 if (RetVT.SimpleTy != MVT::v8i32)
17923 return 0;
17924 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17925 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17926 }
17927 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17928 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17929 }
17930 return 0;
17931}
17932
17933unsigned fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17934 if (RetVT.SimpleTy != MVT::v16i32)
17935 return 0;
17936 if ((Subtarget->hasAVX512())) {
17937 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
17938 }
17939 return 0;
17940}
17941
17942unsigned fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17943 if (RetVT.SimpleTy != MVT::v2i64)
17944 return 0;
17945 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17946 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17947 }
17948 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17949 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
17950 }
17951 return 0;
17952}
17953
17954unsigned fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17955 if (RetVT.SimpleTy != MVT::v4i64)
17956 return 0;
17957 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17958 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17959 }
17960 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17961 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17962 }
17963 return 0;
17964}
17965
17966unsigned fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17967 if (RetVT.SimpleTy != MVT::v8i64)
17968 return 0;
17969 if ((Subtarget->hasAVX512())) {
17970 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17971 }
17972 return 0;
17973}
17974
17975unsigned fastEmit_X86ISD_VSRLV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17976 switch (VT.SimpleTy) {
17977 case MVT::v8i16: return fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(RetVT, Op0, Op1);
17978 case MVT::v16i16: return fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(RetVT, Op0, Op1);
17979 case MVT::v32i16: return fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(RetVT, Op0, Op1);
17980 case MVT::v4i32: return fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(RetVT, Op0, Op1);
17981 case MVT::v8i32: return fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(RetVT, Op0, Op1);
17982 case MVT::v16i32: return fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(RetVT, Op0, Op1);
17983 case MVT::v2i64: return fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(RetVT, Op0, Op1);
17984 case MVT::v4i64: return fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(RetVT, Op0, Op1);
17985 case MVT::v8i64: return fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(RetVT, Op0, Op1);
17986 default: return 0;
17987 }
17988}
17989
17990// Top-level FastEmit function.
17991
17992unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, unsigned Op1) override {
17993 switch (Opcode) {
17994 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
17995 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
17996 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
17997 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
17998 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
17999 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
18000 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
18001 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
18002 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
18003 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
18004 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
18005 case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op1);
18006 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
18007 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
18008 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
18009 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
18010 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
18011 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
18012 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
18013 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
18014 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
18015 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
18016 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
18017 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
18018 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
18019 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
18020 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
18021 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
18022 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
18023 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
18024 case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op1);
18025 case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op1);
18026 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op1);
18027 case X86ISD::BT: return fastEmit_X86ISD_BT_rr(VT, RetVT, Op0, Op1);
18028 case X86ISD::BZHI: return fastEmit_X86ISD_BZHI_rr(VT, RetVT, Op0, Op1);
18029 case X86ISD::CMP: return fastEmit_X86ISD_CMP_rr(VT, RetVT, Op0, Op1);
18030 case X86ISD::COMI: return fastEmit_X86ISD_COMI_rr(VT, RetVT, Op0, Op1);
18031 case X86ISD::CVTNE2PS2BF16: return fastEmit_X86ISD_CVTNE2PS2BF16_rr(VT, RetVT, Op0, Op1);
18032 case X86ISD::FADDS: return fastEmit_X86ISD_FADDS_rr(VT, RetVT, Op0, Op1);
18033 case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op1);
18034 case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op1);
18035 case X86ISD::FCMP: return fastEmit_X86ISD_FCMP_rr(VT, RetVT, Op0, Op1);
18036 case X86ISD::FDIVS: return fastEmit_X86ISD_FDIVS_rr(VT, RetVT, Op0, Op1);
18037 case X86ISD::FGETEXPS: return fastEmit_X86ISD_FGETEXPS_rr(VT, RetVT, Op0, Op1);
18038 case X86ISD::FGETEXPS_SAE: return fastEmit_X86ISD_FGETEXPS_SAE_rr(VT, RetVT, Op0, Op1);
18039 case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op1);
18040 case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op1);
18041 case X86ISD::FMAX: return fastEmit_X86ISD_FMAX_rr(VT, RetVT, Op0, Op1);
18042 case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op1);
18043 case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op1);
18044 case X86ISD::FMAXS_SAE: return fastEmit_X86ISD_FMAXS_SAE_rr(VT, RetVT, Op0, Op1);
18045 case X86ISD::FMAX_SAE: return fastEmit_X86ISD_FMAX_SAE_rr(VT, RetVT, Op0, Op1);
18046 case X86ISD::FMIN: return fastEmit_X86ISD_FMIN_rr(VT, RetVT, Op0, Op1);
18047 case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op1);
18048 case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op1);
18049 case X86ISD::FMINS_SAE: return fastEmit_X86ISD_FMINS_SAE_rr(VT, RetVT, Op0, Op1);
18050 case X86ISD::FMIN_SAE: return fastEmit_X86ISD_FMIN_SAE_rr(VT, RetVT, Op0, Op1);
18051 case X86ISD::FMULS: return fastEmit_X86ISD_FMULS_rr(VT, RetVT, Op0, Op1);
18052 case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op1);
18053 case X86ISD::FP80_ADD: return fastEmit_X86ISD_FP80_ADD_rr(VT, RetVT, Op0, Op1);
18054 case X86ISD::FSQRTS: return fastEmit_X86ISD_FSQRTS_rr(VT, RetVT, Op0, Op1);
18055 case X86ISD::FSUBS: return fastEmit_X86ISD_FSUBS_rr(VT, RetVT, Op0, Op1);
18056 case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op1);
18057 case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op1);
18058 case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op1);
18059 case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op1);
18060 case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op1);
18061 case X86ISD::KORTEST: return fastEmit_X86ISD_KORTEST_rr(VT, RetVT, Op0, Op1);
18062 case X86ISD::KTEST: return fastEmit_X86ISD_KTEST_rr(VT, RetVT, Op0, Op1);
18063 case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op1);
18064 case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op1);
18065 case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op1);
18066 case X86ISD::MOVSH: return fastEmit_X86ISD_MOVSH_rr(VT, RetVT, Op0, Op1);
18067 case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op1);
18068 case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op1);
18069 case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op1);
18070 case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op1);
18071 case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op1);
18072 case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op1);
18073 case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op1);
18074 case X86ISD::PDEP: return fastEmit_X86ISD_PDEP_rr(VT, RetVT, Op0, Op1);
18075 case X86ISD::PEXT: return fastEmit_X86ISD_PEXT_rr(VT, RetVT, Op0, Op1);
18076 case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op1);
18077 case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op1);
18078 case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op1);
18079 case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op1);
18080 case X86ISD::PTEST: return fastEmit_X86ISD_PTEST_rr(VT, RetVT, Op0, Op1);
18081 case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op1);
18082 case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op1);
18083 case X86ISD::SCALEF: return fastEmit_X86ISD_SCALEF_rr(VT, RetVT, Op0, Op1);
18084 case X86ISD::SCALEFS: return fastEmit_X86ISD_SCALEFS_rr(VT, RetVT, Op0, Op1);
18085 case X86ISD::STRICT_FCMP: return fastEmit_X86ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
18086 case X86ISD::STRICT_FCMPS: return fastEmit_X86ISD_STRICT_FCMPS_rr(VT, RetVT, Op0, Op1);
18087 case X86ISD::STRICT_FP80_ADD: return fastEmit_X86ISD_STRICT_FP80_ADD_rr(VT, RetVT, Op0, Op1);
18088 case X86ISD::TESTP: return fastEmit_X86ISD_TESTP_rr(VT, RetVT, Op0, Op1);
18089 case X86ISD::UCOMI: return fastEmit_X86ISD_UCOMI_rr(VT, RetVT, Op0, Op1);
18090 case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op1);
18091 case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op1);
18092 case X86ISD::VFCMULC: return fastEmit_X86ISD_VFCMULC_rr(VT, RetVT, Op0, Op1);
18093 case X86ISD::VFCMULCSH: return fastEmit_X86ISD_VFCMULCSH_rr(VT, RetVT, Op0, Op1);
18094 case X86ISD::VFMULC: return fastEmit_X86ISD_VFMULC_rr(VT, RetVT, Op0, Op1);
18095 case X86ISD::VFMULCSH: return fastEmit_X86ISD_VFMULCSH_rr(VT, RetVT, Op0, Op1);
18096 case X86ISD::VP2INTERSECT: return fastEmit_X86ISD_VP2INTERSECT_rr(VT, RetVT, Op0, Op1);
18097 case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op1);
18098 case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op1);
18099 case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op1);
18100 case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op1);
18101 case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op1);
18102 case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op1);
18103 case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op1);
18104 case X86ISD::VSHLV: return fastEmit_X86ISD_VSHLV_rr(VT, RetVT, Op0, Op1);
18105 case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op1);
18106 case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op1);
18107 case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op1);
18108 case X86ISD::VSRLV: return fastEmit_X86ISD_VSRLV_rr(VT, RetVT, Op0, Op1);
18109 default: return 0;
18110 }
18111}
18112
18113// FastEmit functions for ISD::ADD.
18114
18115unsigned fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18116 if (RetVT.SimpleTy != MVT::i8)
18117 return 0;
18118 if ((Subtarget->hasNDD())) {
18119 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18120 }
18121 if ((!Subtarget->hasNDD())) {
18122 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18123 }
18124 return 0;
18125}
18126
18127unsigned fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18128 if (RetVT.SimpleTy != MVT::i16)
18129 return 0;
18130 if ((Subtarget->hasNDD())) {
18131 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18132 }
18133 if ((!Subtarget->hasNDD())) {
18134 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18135 }
18136 return 0;
18137}
18138
18139unsigned fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18140 if (RetVT.SimpleTy != MVT::i32)
18141 return 0;
18142 if ((Subtarget->hasNDD())) {
18143 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18144 }
18145 if ((!Subtarget->hasNDD())) {
18146 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18147 }
18148 return 0;
18149}
18150
18151unsigned fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18152 switch (VT.SimpleTy) {
18153 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, imm1);
18154 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, imm1);
18155 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, imm1);
18156 default: return 0;
18157 }
18158}
18159
18160// FastEmit functions for ISD::AND.
18161
18162unsigned fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18163 if (RetVT.SimpleTy != MVT::i8)
18164 return 0;
18165 if ((Subtarget->hasNDD())) {
18166 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18167 }
18168 if ((!Subtarget->hasNDD())) {
18169 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18170 }
18171 return 0;
18172}
18173
18174unsigned fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18175 if (RetVT.SimpleTy != MVT::i16)
18176 return 0;
18177 if ((Subtarget->hasNDD())) {
18178 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18179 }
18180 if ((!Subtarget->hasNDD())) {
18181 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18182 }
18183 return 0;
18184}
18185
18186unsigned fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18187 if (RetVT.SimpleTy != MVT::i32)
18188 return 0;
18189 if ((Subtarget->hasNDD())) {
18190 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18191 }
18192 if ((!Subtarget->hasNDD())) {
18193 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18194 }
18195 return 0;
18196}
18197
18198unsigned fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18199 switch (VT.SimpleTy) {
18200 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, imm1);
18201 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, imm1);
18202 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, imm1);
18203 default: return 0;
18204 }
18205}
18206
18207// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
18208
18209unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18210 if (RetVT.SimpleTy != MVT::i32)
18211 return 0;
18212 if ((Subtarget->hasDQI())) {
18213 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDZrr, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18214 }
18215 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
18216 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRDrr, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18217 }
18218 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
18219 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDrr, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18220 }
18221 return 0;
18222}
18223
18224unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18225 if (RetVT.SimpleTy != MVT::i64)
18226 return 0;
18227 if ((Subtarget->hasDQI())) {
18228 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQZrr, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18229 }
18230 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
18231 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRQrr, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18232 }
18233 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
18234 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQrr, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18235 }
18236 return 0;
18237}
18238
18239unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18240 switch (VT.SimpleTy) {
18241 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, imm1);
18242 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, imm1);
18243 default: return 0;
18244 }
18245}
18246
18247// FastEmit functions for ISD::MUL.
18248
18249unsigned fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18250 if (RetVT.SimpleTy != MVT::i16)
18251 return 0;
18252 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL16rri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18253}
18254
18255unsigned fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18256 if (RetVT.SimpleTy != MVT::i32)
18257 return 0;
18258 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL32rri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18259}
18260
18261unsigned fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18262 switch (VT.SimpleTy) {
18263 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, imm1);
18264 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, imm1);
18265 default: return 0;
18266 }
18267}
18268
18269// FastEmit functions for ISD::OR.
18270
18271unsigned fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18272 if (RetVT.SimpleTy != MVT::i8)
18273 return 0;
18274 if ((Subtarget->hasNDD())) {
18275 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18276 }
18277 if ((!Subtarget->hasNDD())) {
18278 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18279 }
18280 return 0;
18281}
18282
18283unsigned fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18284 if (RetVT.SimpleTy != MVT::i16)
18285 return 0;
18286 if ((Subtarget->hasNDD())) {
18287 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18288 }
18289 if ((!Subtarget->hasNDD())) {
18290 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18291 }
18292 return 0;
18293}
18294
18295unsigned fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18296 if (RetVT.SimpleTy != MVT::i32)
18297 return 0;
18298 if ((Subtarget->hasNDD())) {
18299 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18300 }
18301 if ((!Subtarget->hasNDD())) {
18302 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18303 }
18304 return 0;
18305}
18306
18307unsigned fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18308 switch (VT.SimpleTy) {
18309 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, imm1);
18310 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, imm1);
18311 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, imm1);
18312 default: return 0;
18313 }
18314}
18315
18316// FastEmit functions for ISD::ROTL.
18317
18318unsigned fastEmit_ISD_ROTL_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18319 if (RetVT.SimpleTy != MVT::i8)
18320 return 0;
18321 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18322 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18323 }
18324 if ((!Subtarget->hasNDD())) {
18325 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18326 }
18327 return 0;
18328}
18329
18330unsigned fastEmit_ISD_ROTL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18331 if (RetVT.SimpleTy != MVT::i16)
18332 return 0;
18333 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18334 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18335 }
18336 if ((!Subtarget->hasNDD())) {
18337 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18338 }
18339 return 0;
18340}
18341
18342unsigned fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18343 if (RetVT.SimpleTy != MVT::i32)
18344 return 0;
18345 if ((Subtarget->hasFastSHLDRotate())) {
18346 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18347 }
18348 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18349 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18350 }
18351 if ((!Subtarget->hasNDD())) {
18352 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18353 }
18354 return 0;
18355}
18356
18357unsigned fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18358 if (RetVT.SimpleTy != MVT::i64)
18359 return 0;
18360 if ((Subtarget->hasFastSHLDRotate())) {
18361 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18362 }
18363 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18364 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18365 }
18366 if ((!Subtarget->hasNDD())) {
18367 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18368 }
18369 return 0;
18370}
18371
18372unsigned fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18373 switch (VT.SimpleTy) {
18374 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_ri(RetVT, Op0, imm1);
18375 case MVT::i16: return fastEmit_ISD_ROTL_MVT_i16_ri(RetVT, Op0, imm1);
18376 case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, imm1);
18377 case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, imm1);
18378 default: return 0;
18379 }
18380}
18381
18382// FastEmit functions for ISD::ROTR.
18383
18384unsigned fastEmit_ISD_ROTR_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18385 if (RetVT.SimpleTy != MVT::i8)
18386 return 0;
18387 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18388 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18389 }
18390 if ((!Subtarget->hasNDD())) {
18391 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18392 }
18393 return 0;
18394}
18395
18396unsigned fastEmit_ISD_ROTR_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18397 if (RetVT.SimpleTy != MVT::i16)
18398 return 0;
18399 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18400 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18401 }
18402 if ((!Subtarget->hasNDD())) {
18403 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18404 }
18405 return 0;
18406}
18407
18408unsigned fastEmit_ISD_ROTR_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18409 if (RetVT.SimpleTy != MVT::i32)
18410 return 0;
18411 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
18412 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri_EVEX, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18413 }
18414 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
18415 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18416 }
18417 if ((Subtarget->hasFastSHLDRotate())) {
18418 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18419 }
18420 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18421 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18422 }
18423 if ((!Subtarget->hasNDD())) {
18424 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18425 }
18426 return 0;
18427}
18428
18429unsigned fastEmit_ISD_ROTR_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18430 if (RetVT.SimpleTy != MVT::i64)
18431 return 0;
18432 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
18433 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri_EVEX, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18434 }
18435 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
18436 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18437 }
18438 if ((Subtarget->hasFastSHLDRotate())) {
18439 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18440 }
18441 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18442 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18443 }
18444 if ((!Subtarget->hasNDD())) {
18445 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18446 }
18447 return 0;
18448}
18449
18450unsigned fastEmit_ISD_ROTR_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18451 switch (VT.SimpleTy) {
18452 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_ri(RetVT, Op0, imm1);
18453 case MVT::i16: return fastEmit_ISD_ROTR_MVT_i16_ri(RetVT, Op0, imm1);
18454 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri(RetVT, Op0, imm1);
18455 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_ri(RetVT, Op0, imm1);
18456 default: return 0;
18457 }
18458}
18459
18460// FastEmit functions for ISD::SHL.
18461
18462unsigned fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18463 if (RetVT.SimpleTy != MVT::i8)
18464 return 0;
18465 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18466 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18467 }
18468 if ((!Subtarget->hasNDD())) {
18469 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18470 }
18471 return 0;
18472}
18473
18474unsigned fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18475 if (RetVT.SimpleTy != MVT::i16)
18476 return 0;
18477 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18478 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18479 }
18480 if ((!Subtarget->hasNDD())) {
18481 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18482 }
18483 return 0;
18484}
18485
18486unsigned fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18487 if (RetVT.SimpleTy != MVT::i32)
18488 return 0;
18489 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18490 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18491 }
18492 if ((!Subtarget->hasNDD())) {
18493 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18494 }
18495 return 0;
18496}
18497
18498unsigned fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18499 if (RetVT.SimpleTy != MVT::i64)
18500 return 0;
18501 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18502 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18503 }
18504 if ((!Subtarget->hasNDD())) {
18505 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18506 }
18507 return 0;
18508}
18509
18510unsigned fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18511 switch (VT.SimpleTy) {
18512 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, imm1);
18513 case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, imm1);
18514 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
18515 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, imm1);
18516 default: return 0;
18517 }
18518}
18519
18520// FastEmit functions for ISD::SRA.
18521
18522unsigned fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18523 if (RetVT.SimpleTy != MVT::i8)
18524 return 0;
18525 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18526 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18527 }
18528 if ((!Subtarget->hasNDD())) {
18529 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18530 }
18531 return 0;
18532}
18533
18534unsigned fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18535 if (RetVT.SimpleTy != MVT::i16)
18536 return 0;
18537 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18538 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18539 }
18540 if ((!Subtarget->hasNDD())) {
18541 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18542 }
18543 return 0;
18544}
18545
18546unsigned fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18547 if (RetVT.SimpleTy != MVT::i32)
18548 return 0;
18549 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18550 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18551 }
18552 if ((!Subtarget->hasNDD())) {
18553 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18554 }
18555 return 0;
18556}
18557
18558unsigned fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18559 if (RetVT.SimpleTy != MVT::i64)
18560 return 0;
18561 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18562 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18563 }
18564 if ((!Subtarget->hasNDD())) {
18565 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18566 }
18567 return 0;
18568}
18569
18570unsigned fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18571 switch (VT.SimpleTy) {
18572 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, imm1);
18573 case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, imm1);
18574 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, imm1);
18575 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, imm1);
18576 default: return 0;
18577 }
18578}
18579
18580// FastEmit functions for ISD::SRL.
18581
18582unsigned fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18583 if (RetVT.SimpleTy != MVT::i8)
18584 return 0;
18585 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18586 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18587 }
18588 if ((!Subtarget->hasNDD())) {
18589 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18590 }
18591 return 0;
18592}
18593
18594unsigned fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18595 if (RetVT.SimpleTy != MVT::i16)
18596 return 0;
18597 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18598 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18599 }
18600 if ((!Subtarget->hasNDD())) {
18601 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18602 }
18603 return 0;
18604}
18605
18606unsigned fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18607 if (RetVT.SimpleTy != MVT::i32)
18608 return 0;
18609 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18610 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18611 }
18612 if ((!Subtarget->hasNDD())) {
18613 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18614 }
18615 return 0;
18616}
18617
18618unsigned fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18619 if (RetVT.SimpleTy != MVT::i64)
18620 return 0;
18621 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
18622 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18623 }
18624 if ((!Subtarget->hasNDD())) {
18625 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18626 }
18627 return 0;
18628}
18629
18630unsigned fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18631 switch (VT.SimpleTy) {
18632 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, imm1);
18633 case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, imm1);
18634 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, imm1);
18635 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, imm1);
18636 default: return 0;
18637 }
18638}
18639
18640// FastEmit functions for ISD::SUB.
18641
18642unsigned fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18643 if (RetVT.SimpleTy != MVT::i8)
18644 return 0;
18645 if ((Subtarget->hasNDD())) {
18646 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18647 }
18648 if ((!Subtarget->hasNDD())) {
18649 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18650 }
18651 return 0;
18652}
18653
18654unsigned fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18655 if (RetVT.SimpleTy != MVT::i16)
18656 return 0;
18657 if ((Subtarget->hasNDD())) {
18658 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18659 }
18660 if ((!Subtarget->hasNDD())) {
18661 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18662 }
18663 return 0;
18664}
18665
18666unsigned fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18667 if (RetVT.SimpleTy != MVT::i32)
18668 return 0;
18669 if ((Subtarget->hasNDD())) {
18670 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18671 }
18672 if ((!Subtarget->hasNDD())) {
18673 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18674 }
18675 return 0;
18676}
18677
18678unsigned fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18679 switch (VT.SimpleTy) {
18680 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, imm1);
18681 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, imm1);
18682 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, imm1);
18683 default: return 0;
18684 }
18685}
18686
18687// FastEmit functions for ISD::XOR.
18688
18689unsigned fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18690 if (RetVT.SimpleTy != MVT::i8)
18691 return 0;
18692 if ((Subtarget->hasNDD())) {
18693 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18694 }
18695 if ((!Subtarget->hasNDD())) {
18696 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
18697 }
18698 return 0;
18699}
18700
18701unsigned fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18702 if (RetVT.SimpleTy != MVT::i16)
18703 return 0;
18704 if ((Subtarget->hasNDD())) {
18705 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18706 }
18707 if ((!Subtarget->hasNDD())) {
18708 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18709 }
18710 return 0;
18711}
18712
18713unsigned fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18714 if (RetVT.SimpleTy != MVT::i32)
18715 return 0;
18716 if ((Subtarget->hasNDD())) {
18717 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18718 }
18719 if ((!Subtarget->hasNDD())) {
18720 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18721 }
18722 return 0;
18723}
18724
18725unsigned fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18726 switch (VT.SimpleTy) {
18727 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, imm1);
18728 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, imm1);
18729 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, imm1);
18730 default: return 0;
18731 }
18732}
18733
18734// FastEmit functions for X86ISD::BT.
18735
18736unsigned fastEmit_X86ISD_BT_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18737 if (RetVT.SimpleTy != MVT::i32)
18738 return 0;
18739 return fastEmitInst_ri(MachineInstOpcode: X86::BT16ri8, RC: &X86::GR16RegClass, Op0, Imm: imm1);
18740}
18741
18742unsigned fastEmit_X86ISD_BT_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18743 if (RetVT.SimpleTy != MVT::i32)
18744 return 0;
18745 return fastEmitInst_ri(MachineInstOpcode: X86::BT32ri8, RC: &X86::GR32RegClass, Op0, Imm: imm1);
18746}
18747
18748unsigned fastEmit_X86ISD_BT_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18749 if (RetVT.SimpleTy != MVT::i32)
18750 return 0;
18751 return fastEmitInst_ri(MachineInstOpcode: X86::BT64ri8, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18752}
18753
18754unsigned fastEmit_X86ISD_BT_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18755 switch (VT.SimpleTy) {
18756 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_ri(RetVT, Op0, imm1);
18757 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_ri(RetVT, Op0, imm1);
18758 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_ri(RetVT, Op0, imm1);
18759 default: return 0;
18760 }
18761}
18762
18763// Top-level FastEmit function.
18764
18765unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) override {
18766 if (VT == MVT::i64 && Predicate_i64immSExt32(Imm: imm1))
18767 if (unsigned Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, imm1))
18768 return Reg;
18769
18770 switch (Opcode) {
18771 case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, imm1);
18772 case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, imm1);
18773 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
18774 case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, imm1);
18775 case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, imm1);
18776 case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, imm1);
18777 case ISD::ROTR: return fastEmit_ISD_ROTR_ri(VT, RetVT, Op0, imm1);
18778 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
18779 case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, imm1);
18780 case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, imm1);
18781 case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, imm1);
18782 case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, imm1);
18783 case X86ISD::BT: return fastEmit_X86ISD_BT_ri(VT, RetVT, Op0, imm1);
18784 default: return 0;
18785 }
18786}
18787
18788// FastEmit functions for ISD::ADD.
18789
18790unsigned fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18791 if (RetVT.SimpleTy != MVT::i64)
18792 return 0;
18793 if ((Subtarget->hasNDD())) {
18794 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18795 }
18796 if ((!Subtarget->hasNDD())) {
18797 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18798 }
18799 return 0;
18800}
18801
18802unsigned fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18803 switch (VT.SimpleTy) {
18804 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18805 default: return 0;
18806 }
18807}
18808
18809// FastEmit functions for ISD::AND.
18810
18811unsigned fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18812 if (RetVT.SimpleTy != MVT::i64)
18813 return 0;
18814 if ((Subtarget->hasNDD())) {
18815 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18816 }
18817 if ((!Subtarget->hasNDD())) {
18818 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18819 }
18820 return 0;
18821}
18822
18823unsigned fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18824 switch (VT.SimpleTy) {
18825 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18826 default: return 0;
18827 }
18828}
18829
18830// FastEmit functions for ISD::MUL.
18831
18832unsigned fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18833 if (RetVT.SimpleTy != MVT::i64)
18834 return 0;
18835 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL64rri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18836}
18837
18838unsigned fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18839 switch (VT.SimpleTy) {
18840 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18841 default: return 0;
18842 }
18843}
18844
18845// FastEmit functions for ISD::OR.
18846
18847unsigned fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18848 if (RetVT.SimpleTy != MVT::i64)
18849 return 0;
18850 if ((Subtarget->hasNDD())) {
18851 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18852 }
18853 if ((!Subtarget->hasNDD())) {
18854 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18855 }
18856 return 0;
18857}
18858
18859unsigned fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18860 switch (VT.SimpleTy) {
18861 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18862 default: return 0;
18863 }
18864}
18865
18866// FastEmit functions for ISD::SUB.
18867
18868unsigned fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18869 if (RetVT.SimpleTy != MVT::i64)
18870 return 0;
18871 if ((Subtarget->hasNDD())) {
18872 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18873 }
18874 if ((!Subtarget->hasNDD())) {
18875 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18876 }
18877 return 0;
18878}
18879
18880unsigned fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18881 switch (VT.SimpleTy) {
18882 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18883 default: return 0;
18884 }
18885}
18886
18887// FastEmit functions for ISD::XOR.
18888
18889unsigned fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18890 if (RetVT.SimpleTy != MVT::i64)
18891 return 0;
18892 if ((Subtarget->hasNDD())) {
18893 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18894 }
18895 if ((!Subtarget->hasNDD())) {
18896 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
18897 }
18898 return 0;
18899}
18900
18901unsigned fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18902 switch (VT.SimpleTy) {
18903 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18904 default: return 0;
18905 }
18906}
18907
18908// Top-level FastEmit function.
18909
18910unsigned fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
18911 switch (Opcode) {
18912 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18913 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18914 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18915 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18916 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18917 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18918 default: return 0;
18919 }
18920}
18921
18922// FastEmit functions for ISD::Constant.
18923
18924unsigned fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) {
18925 if (RetVT.SimpleTy != MVT::i8)
18926 return 0;
18927 return fastEmitInst_i(MachineInstOpcode: X86::MOV8ri, RC: &X86::GR8RegClass, Imm: imm0);
18928}
18929
18930unsigned fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) {
18931 if (RetVT.SimpleTy != MVT::i16)
18932 return 0;
18933 return fastEmitInst_i(MachineInstOpcode: X86::MOV16ri, RC: &X86::GR16RegClass, Imm: imm0);
18934}
18935
18936unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
18937 if (RetVT.SimpleTy != MVT::i32)
18938 return 0;
18939 return fastEmitInst_i(MachineInstOpcode: X86::MOV32ri, RC: &X86::GR32RegClass, Imm: imm0);
18940}
18941
18942unsigned fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
18943 if (RetVT.SimpleTy != MVT::i64)
18944 return 0;
18945 return fastEmitInst_i(MachineInstOpcode: X86::MOV64ri, RC: &X86::GR64RegClass, Imm: imm0);
18946}
18947
18948unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
18949 switch (VT.SimpleTy) {
18950 case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0);
18951 case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0);
18952 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
18953 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
18954 default: return 0;
18955 }
18956}
18957
18958// Top-level FastEmit function.
18959
18960unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
18961 switch (Opcode) {
18962 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
18963 default: return 0;
18964 }
18965}
18966
18967