1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the X86 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_AndMask64(int64_t Imm) {
12
13 return isMask_64(Value: Imm) && !isUInt<32>(x: Imm);
14
15}
16static bool Predicate_BTRMask64(int64_t Imm) {
17
18 return !isUInt<32>(x: Imm) && !isInt<32>(x: Imm) && isPowerOf2_64(Value: ~Imm);
19
20}
21static bool Predicate_BTCBTSMask64(int64_t Imm) {
22
23 return !isInt<32>(x: Imm) && isPowerOf2_64(Value: Imm);
24
25}
26static bool Predicate_i64immSExt32(int64_t Imm) {
27 return isInt<32>(x: Imm);
28}
29
30
31// FastEmit functions for ISD::ABS.
32
33Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
34 if (RetVT.SimpleTy != MVT::v16i8)
35 return Register();
36 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
37 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ128rr, RC: &X86::VR128XRegClass, Op0);
38 }
39 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
40 return fastEmitInst_r(MachineInstOpcode: X86::PABSBrr, RC: &X86::VR128RegClass, Op0);
41 }
42 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
43 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBrr, RC: &X86::VR128RegClass, Op0);
44 }
45 return Register();
46}
47
48Register fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, Register Op0) {
49 if (RetVT.SimpleTy != MVT::v32i8)
50 return Register();
51 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
52 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ256rr, RC: &X86::VR256XRegClass, Op0);
53 }
54 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
55 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBYrr, RC: &X86::VR256RegClass, Op0);
56 }
57 return Register();
58}
59
60Register fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, Register Op0) {
61 if (RetVT.SimpleTy != MVT::v64i8)
62 return Register();
63 if ((Subtarget->hasBWI())) {
64 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZrr, RC: &X86::VR512RegClass, Op0);
65 }
66 return Register();
67}
68
69Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
70 if (RetVT.SimpleTy != MVT::v8i16)
71 return Register();
72 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
73 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ128rr, RC: &X86::VR128XRegClass, Op0);
74 }
75 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
76 return fastEmitInst_r(MachineInstOpcode: X86::PABSWrr, RC: &X86::VR128RegClass, Op0);
77 }
78 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
79 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWrr, RC: &X86::VR128RegClass, Op0);
80 }
81 return Register();
82}
83
84Register fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, Register Op0) {
85 if (RetVT.SimpleTy != MVT::v16i16)
86 return Register();
87 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
88 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ256rr, RC: &X86::VR256XRegClass, Op0);
89 }
90 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
91 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWYrr, RC: &X86::VR256RegClass, Op0);
92 }
93 return Register();
94}
95
96Register fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, Register Op0) {
97 if (RetVT.SimpleTy != MVT::v32i16)
98 return Register();
99 if ((Subtarget->hasBWI())) {
100 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZrr, RC: &X86::VR512RegClass, Op0);
101 }
102 return Register();
103}
104
105Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
106 if (RetVT.SimpleTy != MVT::v4i32)
107 return Register();
108 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
109 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ128rr, RC: &X86::VR128XRegClass, Op0);
110 }
111 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
112 return fastEmitInst_r(MachineInstOpcode: X86::PABSDrr, RC: &X86::VR128RegClass, Op0);
113 }
114 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
115 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDrr, RC: &X86::VR128RegClass, Op0);
116 }
117 return Register();
118}
119
120Register fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, Register Op0) {
121 if (RetVT.SimpleTy != MVT::v8i32)
122 return Register();
123 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
124 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ256rr, RC: &X86::VR256XRegClass, Op0);
125 }
126 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
127 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDYrr, RC: &X86::VR256RegClass, Op0);
128 }
129 return Register();
130}
131
132Register fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, Register Op0) {
133 if (RetVT.SimpleTy != MVT::v16i32)
134 return Register();
135 if ((Subtarget->hasAVX512())) {
136 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZrr, RC: &X86::VR512RegClass, Op0);
137 }
138 return Register();
139}
140
141Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
142 if (RetVT.SimpleTy != MVT::v2i64)
143 return Register();
144 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
145 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ128rr, RC: &X86::VR128XRegClass, Op0);
146 }
147 return Register();
148}
149
150Register fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, Register Op0) {
151 if (RetVT.SimpleTy != MVT::v4i64)
152 return Register();
153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
154 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ256rr, RC: &X86::VR256XRegClass, Op0);
155 }
156 return Register();
157}
158
159Register fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, Register Op0) {
160 if (RetVT.SimpleTy != MVT::v8i64)
161 return Register();
162 if ((Subtarget->hasAVX512())) {
163 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZrr, RC: &X86::VR512RegClass, Op0);
164 }
165 return Register();
166}
167
168Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
169 switch (VT.SimpleTy) {
170 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
171 case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0);
172 case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0);
173 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
174 case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0);
175 case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0);
176 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
177 case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0);
178 case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0);
179 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
180 case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0);
181 case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0);
182 default: return Register();
183 }
184}
185
186// FastEmit functions for ISD::ANY_EXTEND.
187
188Register fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
189 if (RetVT.SimpleTy != MVT::i32)
190 return Register();
191 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
192}
193
194Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
195 switch (VT.SimpleTy) {
196 case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0);
197 default: return Register();
198 }
199}
200
201// FastEmit functions for ISD::BITCAST.
202
203Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
204 if (RetVT.SimpleTy != MVT::f32)
205 return Register();
206 if ((Subtarget->hasAVX512())) {
207 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSZrr, RC: &X86::FR32XRegClass, Op0);
208 }
209 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
210 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
211 }
212 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
213 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
214 }
215 return Register();
216}
217
218Register fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Register Op0) {
219 if ((Subtarget->hasAVX512())) {
220 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDZrr, RC: &X86::FR64XRegClass, Op0);
221 }
222 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
223 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toSDrr, RC: &X86::FR64RegClass, Op0);
224 }
225 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
226 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDrr, RC: &X86::FR64RegClass, Op0);
227 }
228 return Register();
229}
230
231Register fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Register Op0) {
232 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
233 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64to64rr, RC: &X86::VR64RegClass, Op0);
234 }
235 return Register();
236}
237
238Register fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, Register Op0) {
239switch (RetVT.SimpleTy) {
240 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0);
241 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0);
242 default: return Register();
243}
244}
245
246Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
247 if (RetVT.SimpleTy != MVT::i32)
248 return Register();
249 if ((Subtarget->hasAVX512())) {
250 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIZrr, RC: &X86::GR32RegClass, Op0);
251 }
252 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
253 return fastEmitInst_r(MachineInstOpcode: X86::MOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
254 }
255 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
256 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
257 }
258 return Register();
259}
260
261Register fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Register Op0) {
262 if ((Subtarget->hasAVX512())) {
263 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64Zrr, RC: &X86::GR64RegClass, Op0);
264 }
265 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
266 return fastEmitInst_r(MachineInstOpcode: X86::MOVSDto64rr, RC: &X86::GR64RegClass, Op0);
267 }
268 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
269 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64rr, RC: &X86::GR64RegClass, Op0);
270 }
271 return Register();
272}
273
274Register fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Register Op0) {
275 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
276 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVFR642Qrr, RC: &X86::VR64RegClass, Op0);
277 }
278 return Register();
279}
280
281Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
282switch (RetVT.SimpleTy) {
283 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0);
284 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0);
285 default: return Register();
286}
287}
288
289Register fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Register Op0) {
290 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
291 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64from64rr, RC: &X86::GR64RegClass, Op0);
292 }
293 return Register();
294}
295
296Register fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Register Op0) {
297 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
298 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2FR64rr, RC: &X86::FR64RegClass, Op0);
299 }
300 return Register();
301}
302
303Register fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, Register Op0) {
304switch (RetVT.SimpleTy) {
305 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0);
306 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0);
307 default: return Register();
308}
309}
310
311Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
312 switch (VT.SimpleTy) {
313 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
314 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0);
315 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
316 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
317 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0);
318 default: return Register();
319 }
320}
321
322// FastEmit functions for ISD::BRIND.
323
324Register fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, Register Op0) {
325 if (RetVT.SimpleTy != MVT::isVoid)
326 return Register();
327 if ((!Subtarget->is64Bit())) {
328 return fastEmitInst_r(MachineInstOpcode: X86::JMP16r, RC: &X86::GR16RegClass, Op0);
329 }
330 return Register();
331}
332
333Register fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
334 if (RetVT.SimpleTy != MVT::isVoid)
335 return Register();
336 if ((!Subtarget->is64Bit())) {
337 return fastEmitInst_r(MachineInstOpcode: X86::JMP32r, RC: &X86::GR32RegClass, Op0);
338 }
339 return Register();
340}
341
342Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
343 if (RetVT.SimpleTy != MVT::isVoid)
344 return Register();
345 if ((Subtarget->is64Bit())) {
346 return fastEmitInst_r(MachineInstOpcode: X86::JMP64r, RC: &X86::GR64RegClass, Op0);
347 }
348 return Register();
349}
350
351Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
352 switch (VT.SimpleTy) {
353 case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0);
354 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
355 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
356 default: return Register();
357 }
358}
359
360// FastEmit functions for ISD::BSWAP.
361
362Register fastEmit_ISD_BSWAP_MVT_i16_r(MVT RetVT, Register Op0) {
363 if (RetVT.SimpleTy != MVT::i16)
364 return Register();
365 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
366 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE16rr, RC: &X86::GR16RegClass, Op0);
367 }
368 return Register();
369}
370
371Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
372 if (RetVT.SimpleTy != MVT::i32)
373 return Register();
374 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
375 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE32rr, RC: &X86::GR32RegClass, Op0);
376 }
377 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
378 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP32r, RC: &X86::GR32RegClass, Op0);
379 }
380 return Register();
381}
382
383Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
384 if (RetVT.SimpleTy != MVT::i64)
385 return Register();
386 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
387 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE64rr, RC: &X86::GR64RegClass, Op0);
388 }
389 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
390 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP64r, RC: &X86::GR64RegClass, Op0);
391 }
392 return Register();
393}
394
395Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
396 switch (VT.SimpleTy) {
397 case MVT::i16: return fastEmit_ISD_BSWAP_MVT_i16_r(RetVT, Op0);
398 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
399 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
400 default: return Register();
401 }
402}
403
404// FastEmit functions for ISD::CTLZ.
405
406Register fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, Register Op0) {
407 if (RetVT.SimpleTy != MVT::i16)
408 return Register();
409 if ((Subtarget->hasLZCNT())) {
410 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT16rr, RC: &X86::GR16RegClass, Op0);
411 }
412 return Register();
413}
414
415Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
416 if (RetVT.SimpleTy != MVT::i32)
417 return Register();
418 if ((Subtarget->hasLZCNT())) {
419 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT32rr, RC: &X86::GR32RegClass, Op0);
420 }
421 return Register();
422}
423
424Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
425 if (RetVT.SimpleTy != MVT::i64)
426 return Register();
427 if ((Subtarget->hasLZCNT())) {
428 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT64rr, RC: &X86::GR64RegClass, Op0);
429 }
430 return Register();
431}
432
433Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
434 if (RetVT.SimpleTy != MVT::v4i32)
435 return Register();
436 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
437 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
438 }
439 return Register();
440}
441
442Register fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, Register Op0) {
443 if (RetVT.SimpleTy != MVT::v8i32)
444 return Register();
445 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
446 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
447 }
448 return Register();
449}
450
451Register fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, Register Op0) {
452 if (RetVT.SimpleTy != MVT::v16i32)
453 return Register();
454 if ((Subtarget->hasCDI())) {
455 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZrr, RC: &X86::VR512RegClass, Op0);
456 }
457 return Register();
458}
459
460Register fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, Register Op0) {
461 if (RetVT.SimpleTy != MVT::v2i64)
462 return Register();
463 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
464 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
465 }
466 return Register();
467}
468
469Register fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, Register Op0) {
470 if (RetVT.SimpleTy != MVT::v4i64)
471 return Register();
472 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
473 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
474 }
475 return Register();
476}
477
478Register fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, Register Op0) {
479 if (RetVT.SimpleTy != MVT::v8i64)
480 return Register();
481 if ((Subtarget->hasCDI())) {
482 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZrr, RC: &X86::VR512RegClass, Op0);
483 }
484 return Register();
485}
486
487Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
488 switch (VT.SimpleTy) {
489 case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0);
490 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
491 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
492 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
493 case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0);
494 case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0);
495 case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0);
496 case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0);
497 case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0);
498 default: return Register();
499 }
500}
501
502// FastEmit functions for ISD::CTPOP.
503
504Register fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, Register Op0) {
505 if (RetVT.SimpleTy != MVT::i16)
506 return Register();
507 if ((Subtarget->hasPOPCNT())) {
508 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT16rr, RC: &X86::GR16RegClass, Op0);
509 }
510 return Register();
511}
512
513Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
514 if (RetVT.SimpleTy != MVT::i32)
515 return Register();
516 if ((Subtarget->hasPOPCNT())) {
517 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT32rr, RC: &X86::GR32RegClass, Op0);
518 }
519 return Register();
520}
521
522Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
523 if (RetVT.SimpleTy != MVT::i64)
524 return Register();
525 if ((Subtarget->hasPOPCNT())) {
526 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT64rr, RC: &X86::GR64RegClass, Op0);
527 }
528 return Register();
529}
530
531Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
532 if (RetVT.SimpleTy != MVT::v16i8)
533 return Register();
534 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
535 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ128rr, RC: &X86::VR128XRegClass, Op0);
536 }
537 return Register();
538}
539
540Register fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, Register Op0) {
541 if (RetVT.SimpleTy != MVT::v32i8)
542 return Register();
543 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
544 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ256rr, RC: &X86::VR256XRegClass, Op0);
545 }
546 return Register();
547}
548
549Register fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, Register Op0) {
550 if (RetVT.SimpleTy != MVT::v64i8)
551 return Register();
552 if ((Subtarget->hasBITALG())) {
553 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZrr, RC: &X86::VR512RegClass, Op0);
554 }
555 return Register();
556}
557
558Register fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, Register Op0) {
559 if (RetVT.SimpleTy != MVT::v8i16)
560 return Register();
561 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
562 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ128rr, RC: &X86::VR128XRegClass, Op0);
563 }
564 return Register();
565}
566
567Register fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, Register Op0) {
568 if (RetVT.SimpleTy != MVT::v16i16)
569 return Register();
570 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
571 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ256rr, RC: &X86::VR256XRegClass, Op0);
572 }
573 return Register();
574}
575
576Register fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, Register Op0) {
577 if (RetVT.SimpleTy != MVT::v32i16)
578 return Register();
579 if ((Subtarget->hasBITALG())) {
580 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZrr, RC: &X86::VR512RegClass, Op0);
581 }
582 return Register();
583}
584
585Register fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, Register Op0) {
586 if (RetVT.SimpleTy != MVT::v4i32)
587 return Register();
588 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
589 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
590 }
591 return Register();
592}
593
594Register fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, Register Op0) {
595 if (RetVT.SimpleTy != MVT::v8i32)
596 return Register();
597 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
598 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
599 }
600 return Register();
601}
602
603Register fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, Register Op0) {
604 if (RetVT.SimpleTy != MVT::v16i32)
605 return Register();
606 if ((Subtarget->hasVPOPCNTDQ())) {
607 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZrr, RC: &X86::VR512RegClass, Op0);
608 }
609 return Register();
610}
611
612Register fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, Register Op0) {
613 if (RetVT.SimpleTy != MVT::v2i64)
614 return Register();
615 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
616 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
617 }
618 return Register();
619}
620
621Register fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, Register Op0) {
622 if (RetVT.SimpleTy != MVT::v4i64)
623 return Register();
624 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
625 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
626 }
627 return Register();
628}
629
630Register fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, Register Op0) {
631 if (RetVT.SimpleTy != MVT::v8i64)
632 return Register();
633 if ((Subtarget->hasVPOPCNTDQ())) {
634 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZrr, RC: &X86::VR512RegClass, Op0);
635 }
636 return Register();
637}
638
639Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
640 switch (VT.SimpleTy) {
641 case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0);
642 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
643 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
644 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
645 case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0);
646 case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0);
647 case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0);
648 case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0);
649 case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0);
650 case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0);
651 case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0);
652 case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0);
653 case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0);
654 case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0);
655 case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0);
656 default: return Register();
657 }
658}
659
660// FastEmit functions for ISD::CTTZ.
661
662Register fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, Register Op0) {
663 if (RetVT.SimpleTy != MVT::i16)
664 return Register();
665 if ((Subtarget->hasBMI())) {
666 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT16rr, RC: &X86::GR16RegClass, Op0);
667 }
668 return Register();
669}
670
671Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
672 if (RetVT.SimpleTy != MVT::i32)
673 return Register();
674 if ((Subtarget->hasBMI())) {
675 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT32rr, RC: &X86::GR32RegClass, Op0);
676 }
677 return Register();
678}
679
680Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
681 if (RetVT.SimpleTy != MVT::i64)
682 return Register();
683 if ((Subtarget->hasBMI())) {
684 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT64rr, RC: &X86::GR64RegClass, Op0);
685 }
686 return Register();
687}
688
689Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
690 switch (VT.SimpleTy) {
691 case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0);
692 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
693 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
694 default: return Register();
695 }
696}
697
698// FastEmit functions for ISD::FABS.
699
700Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
701 if (RetVT.SimpleTy != MVT::f32)
702 return Register();
703 if ((!Subtarget->hasSSE1())) {
704 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp32, RC: &X86::RFP32RegClass, Op0);
705 }
706 return Register();
707}
708
709Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
710 if (RetVT.SimpleTy != MVT::f64)
711 return Register();
712 if ((!Subtarget->hasSSE2())) {
713 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp64, RC: &X86::RFP64RegClass, Op0);
714 }
715 return Register();
716}
717
718Register fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, Register Op0) {
719 if (RetVT.SimpleTy != MVT::f80)
720 return Register();
721 if ((Subtarget->hasX87())) {
722 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp80, RC: &X86::RFP80RegClass, Op0);
723 }
724 return Register();
725}
726
727Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
728 switch (VT.SimpleTy) {
729 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
730 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
731 case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0);
732 default: return Register();
733 }
734}
735
736// FastEmit functions for ISD::FNEG.
737
738Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
739 if (RetVT.SimpleTy != MVT::f32)
740 return Register();
741 if ((!Subtarget->hasSSE1())) {
742 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp32, RC: &X86::RFP32RegClass, Op0);
743 }
744 return Register();
745}
746
747Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
748 if (RetVT.SimpleTy != MVT::f64)
749 return Register();
750 if ((!Subtarget->hasSSE2())) {
751 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp64, RC: &X86::RFP64RegClass, Op0);
752 }
753 return Register();
754}
755
756Register fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, Register Op0) {
757 if (RetVT.SimpleTy != MVT::f80)
758 return Register();
759 if ((Subtarget->hasX87())) {
760 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp80, RC: &X86::RFP80RegClass, Op0);
761 }
762 return Register();
763}
764
765Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
766 switch (VT.SimpleTy) {
767 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
768 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
769 case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0);
770 default: return Register();
771 }
772}
773
774// FastEmit functions for ISD::FP_EXTEND.
775
776Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
777 if (RetVT.SimpleTy != MVT::f64)
778 return Register();
779 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
780 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
781 }
782 return Register();
783}
784
785Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Register Op0) {
786 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
787 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
788 }
789 return Register();
790}
791
792Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Register Op0) {
793 if ((Subtarget->hasFP16())) {
794 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
795 }
796 return Register();
797}
798
799Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(MVT RetVT, Register Op0) {
800switch (RetVT.SimpleTy) {
801 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
802 case MVT::v8f64: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
803 default: return Register();
804}
805}
806
807Register fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(MVT RetVT, Register Op0) {
808 if (RetVT.SimpleTy != MVT::v16f32)
809 return Register();
810 if ((Subtarget->hasFP16())) {
811 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
812 }
813 return Register();
814}
815
816Register fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, Register Op0) {
817 if (RetVT.SimpleTy != MVT::v4f64)
818 return Register();
819 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
820 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
821 }
822 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
823 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
824 }
825 return Register();
826}
827
828Register fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, Register Op0) {
829 if (RetVT.SimpleTy != MVT::v8f64)
830 return Register();
831 if ((Subtarget->hasAVX512())) {
832 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
833 }
834 return Register();
835}
836
837Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
838 switch (VT.SimpleTy) {
839 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
840 case MVT::v8f16: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
841 case MVT::v16f16: return fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
842 case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
843 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
844 default: return Register();
845 }
846}
847
848// FastEmit functions for ISD::FP_ROUND.
849
850Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
851 if (RetVT.SimpleTy != MVT::f32)
852 return Register();
853 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
854 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
855 }
856 return Register();
857}
858
859Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
860 switch (VT.SimpleTy) {
861 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
862 default: return Register();
863 }
864}
865
866// FastEmit functions for ISD::FP_TO_SINT.
867
868Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
869 if ((Subtarget->hasFP16())) {
870 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
871 }
872 return Register();
873}
874
875Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
876 if ((Subtarget->hasFP16())) {
877 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
878 }
879 return Register();
880}
881
882Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
883switch (RetVT.SimpleTy) {
884 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
885 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
886 default: return Register();
887}
888}
889
890Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
891 if ((Subtarget->hasAVX512())) {
892 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
893 }
894 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
895 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
896 }
897 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
898 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
899 }
900 return Register();
901}
902
903Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
904 if ((Subtarget->hasAVX512())) {
905 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
906 }
907 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
908 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
909 }
910 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
911 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
912 }
913 return Register();
914}
915
916Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
917switch (RetVT.SimpleTy) {
918 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
919 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
920 default: return Register();
921}
922}
923
924Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
925 if ((Subtarget->hasAVX512())) {
926 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
927 }
928 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
929 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
930 }
931 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
932 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
933 }
934 return Register();
935}
936
937Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
938 if ((Subtarget->hasAVX512())) {
939 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
940 }
941 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
942 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
943 }
944 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
945 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
946 }
947 return Register();
948}
949
950Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
951switch (RetVT.SimpleTy) {
952 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
953 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
954 default: return Register();
955}
956}
957
958Register fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
959 if (RetVT.SimpleTy != MVT::v4i32)
960 return Register();
961 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
963 }
964 return Register();
965}
966
967Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
968 switch (VT.SimpleTy) {
969 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
970 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
971 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
972 case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
973 default: return Register();
974 }
975}
976
977// FastEmit functions for ISD::FP_TO_UINT.
978
979Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
980 if ((Subtarget->hasFP16())) {
981 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
982 }
983 return Register();
984}
985
986Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
987 if ((Subtarget->hasFP16())) {
988 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
989 }
990 return Register();
991}
992
993Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
994switch (RetVT.SimpleTy) {
995 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
996 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
997 default: return Register();
998}
999}
1000
1001Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
1002 if ((Subtarget->hasAVX512())) {
1003 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
1004 }
1005 return Register();
1006}
1007
1008Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
1009 if ((Subtarget->hasAVX512())) {
1010 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1011 }
1012 return Register();
1013}
1014
1015Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
1016switch (RetVT.SimpleTy) {
1017 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
1018 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
1019 default: return Register();
1020}
1021}
1022
1023Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
1024 if ((Subtarget->hasAVX512())) {
1025 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
1026 }
1027 return Register();
1028}
1029
1030Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
1031 if ((Subtarget->hasAVX512())) {
1032 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1033 }
1034 return Register();
1035}
1036
1037Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
1038switch (RetVT.SimpleTy) {
1039 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
1040 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
1041 default: return Register();
1042}
1043}
1044
1045Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
1046 switch (VT.SimpleTy) {
1047 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
1048 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
1049 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
1050 default: return Register();
1051 }
1052}
1053
1054// FastEmit functions for ISD::FSQRT.
1055
1056Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
1057 if (RetVT.SimpleTy != MVT::f32)
1058 return Register();
1059 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1060 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
1061 }
1062 if ((!Subtarget->hasSSE1())) {
1063 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
1064 }
1065 return Register();
1066}
1067
1068Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
1069 if (RetVT.SimpleTy != MVT::f64)
1070 return Register();
1071 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1072 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
1073 }
1074 if ((!Subtarget->hasSSE2())) {
1075 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
1076 }
1077 return Register();
1078}
1079
1080Register fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, Register Op0) {
1081 if (RetVT.SimpleTy != MVT::f80)
1082 return Register();
1083 if ((Subtarget->hasX87())) {
1084 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
1085 }
1086 return Register();
1087}
1088
1089Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1090 if (RetVT.SimpleTy != MVT::v8f16)
1091 return Register();
1092 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1093 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
1094 }
1095 return Register();
1096}
1097
1098Register fastEmit_ISD_FSQRT_MVT_v16f16_r(MVT RetVT, Register Op0) {
1099 if (RetVT.SimpleTy != MVT::v16f16)
1100 return Register();
1101 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1102 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
1103 }
1104 return Register();
1105}
1106
1107Register fastEmit_ISD_FSQRT_MVT_v32f16_r(MVT RetVT, Register Op0) {
1108 if (RetVT.SimpleTy != MVT::v32f16)
1109 return Register();
1110 if ((Subtarget->hasFP16())) {
1111 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
1112 }
1113 return Register();
1114}
1115
1116Register fastEmit_ISD_FSQRT_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1117 if (RetVT.SimpleTy != MVT::v8bf16)
1118 return Register();
1119 if ((Subtarget->hasAVX10_2())) {
1120 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
1121 }
1122 return Register();
1123}
1124
1125Register fastEmit_ISD_FSQRT_MVT_v16bf16_r(MVT RetVT, Register Op0) {
1126 if (RetVT.SimpleTy != MVT::v16bf16)
1127 return Register();
1128 if ((Subtarget->hasAVX10_2())) {
1129 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
1130 }
1131 return Register();
1132}
1133
1134Register fastEmit_ISD_FSQRT_MVT_v32bf16_r(MVT RetVT, Register Op0) {
1135 if (RetVT.SimpleTy != MVT::v32bf16)
1136 return Register();
1137 if ((Subtarget->hasAVX10_2_512())) {
1138 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
1139 }
1140 return Register();
1141}
1142
1143Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1144 if (RetVT.SimpleTy != MVT::v4f32)
1145 return Register();
1146 if ((Subtarget->hasVLX())) {
1147 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
1148 }
1149 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1150 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
1151 }
1152 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1153 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
1154 }
1155 return Register();
1156}
1157
1158Register fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1159 if (RetVT.SimpleTy != MVT::v8f32)
1160 return Register();
1161 if ((Subtarget->hasVLX())) {
1162 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
1163 }
1164 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1165 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
1166 }
1167 return Register();
1168}
1169
1170Register fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, Register Op0) {
1171 if (RetVT.SimpleTy != MVT::v16f32)
1172 return Register();
1173 if ((Subtarget->hasAVX512())) {
1174 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
1175 }
1176 return Register();
1177}
1178
1179Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1180 if (RetVT.SimpleTy != MVT::v2f64)
1181 return Register();
1182 if ((Subtarget->hasVLX())) {
1183 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
1184 }
1185 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1186 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
1187 }
1188 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1189 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
1190 }
1191 return Register();
1192}
1193
1194Register fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1195 if (RetVT.SimpleTy != MVT::v4f64)
1196 return Register();
1197 if ((Subtarget->hasVLX())) {
1198 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
1199 }
1200 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1201 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
1202 }
1203 return Register();
1204}
1205
1206Register fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1207 if (RetVT.SimpleTy != MVT::v8f64)
1208 return Register();
1209 if ((Subtarget->hasAVX512())) {
1210 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
1211 }
1212 return Register();
1213}
1214
1215Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
1216 switch (VT.SimpleTy) {
1217 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
1218 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
1219 case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0);
1220 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
1221 case MVT::v16f16: return fastEmit_ISD_FSQRT_MVT_v16f16_r(RetVT, Op0);
1222 case MVT::v32f16: return fastEmit_ISD_FSQRT_MVT_v32f16_r(RetVT, Op0);
1223 case MVT::v8bf16: return fastEmit_ISD_FSQRT_MVT_v8bf16_r(RetVT, Op0);
1224 case MVT::v16bf16: return fastEmit_ISD_FSQRT_MVT_v16bf16_r(RetVT, Op0);
1225 case MVT::v32bf16: return fastEmit_ISD_FSQRT_MVT_v32bf16_r(RetVT, Op0);
1226 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
1227 case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0);
1228 case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0);
1229 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
1230 case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0);
1231 case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0);
1232 default: return Register();
1233 }
1234}
1235
1236// FastEmit functions for ISD::LLRINT.
1237
1238Register fastEmit_ISD_LLRINT_MVT_f16_r(MVT RetVT, Register Op0) {
1239 if (RetVT.SimpleTy != MVT::i64)
1240 return Register();
1241 if ((Subtarget->hasFP16()) && (Subtarget->is64Bit())) {
1242 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1243 }
1244 return Register();
1245}
1246
1247Register fastEmit_ISD_LLRINT_MVT_f32_r(MVT RetVT, Register Op0) {
1248 if (RetVT.SimpleTy != MVT::i64)
1249 return Register();
1250 if ((Subtarget->hasAVX512())) {
1251 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1252 }
1253 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1254 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1255 }
1256 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1257 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1258 }
1259 return Register();
1260}
1261
1262Register fastEmit_ISD_LLRINT_MVT_f64_r(MVT RetVT, Register Op0) {
1263 if (RetVT.SimpleTy != MVT::i64)
1264 return Register();
1265 if ((Subtarget->hasAVX512())) {
1266 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1267 }
1268 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1269 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1270 }
1271 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1272 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1273 }
1274 return Register();
1275}
1276
1277Register fastEmit_ISD_LLRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1278 if (RetVT.SimpleTy != MVT::v8i64)
1279 return Register();
1280 if ((Subtarget->hasFP16())) {
1281 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
1282 }
1283 return Register();
1284}
1285
1286Register fastEmit_ISD_LLRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1287 if (RetVT.SimpleTy != MVT::v4i64)
1288 return Register();
1289 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1290 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1291 }
1292 return Register();
1293}
1294
1295Register fastEmit_ISD_LLRINT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1296 if (RetVT.SimpleTy != MVT::v8i64)
1297 return Register();
1298 if ((Subtarget->hasDQI())) {
1299 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1300 }
1301 return Register();
1302}
1303
1304Register fastEmit_ISD_LLRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1305 if (RetVT.SimpleTy != MVT::v2i64)
1306 return Register();
1307 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1308 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1309 }
1310 return Register();
1311}
1312
1313Register fastEmit_ISD_LLRINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1314 if (RetVT.SimpleTy != MVT::v4i64)
1315 return Register();
1316 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1317 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1318 }
1319 return Register();
1320}
1321
1322Register fastEmit_ISD_LLRINT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1323 if (RetVT.SimpleTy != MVT::v8i64)
1324 return Register();
1325 if ((Subtarget->hasDQI())) {
1326 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1327 }
1328 return Register();
1329}
1330
1331Register fastEmit_ISD_LLRINT_r(MVT VT, MVT RetVT, Register Op0) {
1332 switch (VT.SimpleTy) {
1333 case MVT::f16: return fastEmit_ISD_LLRINT_MVT_f16_r(RetVT, Op0);
1334 case MVT::f32: return fastEmit_ISD_LLRINT_MVT_f32_r(RetVT, Op0);
1335 case MVT::f64: return fastEmit_ISD_LLRINT_MVT_f64_r(RetVT, Op0);
1336 case MVT::v8f16: return fastEmit_ISD_LLRINT_MVT_v8f16_r(RetVT, Op0);
1337 case MVT::v4f32: return fastEmit_ISD_LLRINT_MVT_v4f32_r(RetVT, Op0);
1338 case MVT::v8f32: return fastEmit_ISD_LLRINT_MVT_v8f32_r(RetVT, Op0);
1339 case MVT::v2f64: return fastEmit_ISD_LLRINT_MVT_v2f64_r(RetVT, Op0);
1340 case MVT::v4f64: return fastEmit_ISD_LLRINT_MVT_v4f64_r(RetVT, Op0);
1341 case MVT::v8f64: return fastEmit_ISD_LLRINT_MVT_v8f64_r(RetVT, Op0);
1342 default: return Register();
1343 }
1344}
1345
1346// FastEmit functions for ISD::LRINT.
1347
1348Register fastEmit_ISD_LRINT_MVT_f16_MVT_i32_r(Register Op0) {
1349 if ((Subtarget->hasFP16())) {
1350 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
1351 }
1352 return Register();
1353}
1354
1355Register fastEmit_ISD_LRINT_MVT_f16_MVT_i64_r(Register Op0) {
1356 if ((Subtarget->hasFP16()) && (Subtarget->is64Bit())) {
1357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1358 }
1359 return Register();
1360}
1361
1362Register fastEmit_ISD_LRINT_MVT_f16_r(MVT RetVT, Register Op0) {
1363switch (RetVT.SimpleTy) {
1364 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f16_MVT_i32_r(Op0);
1365 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f16_MVT_i64_r(Op0);
1366 default: return Register();
1367}
1368}
1369
1370Register fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Register Op0) {
1371 if ((Subtarget->hasAVX512())) {
1372 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
1373 }
1374 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1375 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1376 }
1377 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1378 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1379 }
1380 return Register();
1381}
1382
1383Register fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Register Op0) {
1384 if ((Subtarget->hasAVX512())) {
1385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1386 }
1387 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1388 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1389 }
1390 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1391 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1392 }
1393 return Register();
1394}
1395
1396Register fastEmit_ISD_LRINT_MVT_f32_r(MVT RetVT, Register Op0) {
1397switch (RetVT.SimpleTy) {
1398 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Op0);
1399 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Op0);
1400 default: return Register();
1401}
1402}
1403
1404Register fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Register Op0) {
1405 if ((Subtarget->hasAVX512())) {
1406 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
1407 }
1408 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1409 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1410 }
1411 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1412 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1413 }
1414 return Register();
1415}
1416
1417Register fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Register Op0) {
1418 if ((Subtarget->hasAVX512())) {
1419 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1420 }
1421 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1422 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1423 }
1424 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1425 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1426 }
1427 return Register();
1428}
1429
1430Register fastEmit_ISD_LRINT_MVT_f64_r(MVT RetVT, Register Op0) {
1431switch (RetVT.SimpleTy) {
1432 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Op0);
1433 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Op0);
1434 default: return Register();
1435}
1436}
1437
1438Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i16_r(Register Op0) {
1439 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1440 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
1441 }
1442 return Register();
1443}
1444
1445Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i32_r(Register Op0) {
1446 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1447 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1448 }
1449 return Register();
1450}
1451
1452Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i64_r(Register Op0) {
1453 if ((Subtarget->hasFP16())) {
1454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
1455 }
1456 return Register();
1457}
1458
1459Register fastEmit_ISD_LRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1460switch (RetVT.SimpleTy) {
1461 case MVT::v8i16: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i16_r(Op0);
1462 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i32_r(Op0);
1463 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i64_r(Op0);
1464 default: return Register();
1465}
1466}
1467
1468Register fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i16_r(Register Op0) {
1469 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1470 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
1471 }
1472 return Register();
1473}
1474
1475Register fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i32_r(Register Op0) {
1476 if ((Subtarget->hasFP16())) {
1477 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
1478 }
1479 return Register();
1480}
1481
1482Register fastEmit_ISD_LRINT_MVT_v16f16_r(MVT RetVT, Register Op0) {
1483switch (RetVT.SimpleTy) {
1484 case MVT::v16i16: return fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i16_r(Op0);
1485 case MVT::v16i32: return fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i32_r(Op0);
1486 default: return Register();
1487}
1488}
1489
1490Register fastEmit_ISD_LRINT_MVT_v32f16_r(MVT RetVT, Register Op0) {
1491 if (RetVT.SimpleTy != MVT::v32i16)
1492 return Register();
1493 if ((Subtarget->hasFP16())) {
1494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
1495 }
1496 return Register();
1497}
1498
1499Register fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Register Op0) {
1500 if ((Subtarget->hasVLX())) {
1501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
1502 }
1503 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1504 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1505 }
1506 if ((Subtarget->hasAVX())) {
1507 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1508 }
1509 return Register();
1510}
1511
1512Register fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Register Op0) {
1513 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1514 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1515 }
1516 return Register();
1517}
1518
1519Register fastEmit_ISD_LRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1520switch (RetVT.SimpleTy) {
1521 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Op0);
1522 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Op0);
1523 default: return Register();
1524}
1525}
1526
1527Register fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Register Op0) {
1528 if ((Subtarget->hasVLX())) {
1529 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1530 }
1531 if ((Subtarget->hasAVX())) {
1532 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
1533 }
1534 return Register();
1535}
1536
1537Register fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Register Op0) {
1538 if ((Subtarget->hasDQI())) {
1539 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1540 }
1541 return Register();
1542}
1543
1544Register fastEmit_ISD_LRINT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1545switch (RetVT.SimpleTy) {
1546 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Op0);
1547 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Op0);
1548 default: return Register();
1549}
1550}
1551
1552Register fastEmit_ISD_LRINT_MVT_v16f32_r(MVT RetVT, Register Op0) {
1553 if (RetVT.SimpleTy != MVT::v16i32)
1554 return Register();
1555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
1556}
1557
1558Register fastEmit_ISD_LRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1559 if (RetVT.SimpleTy != MVT::v2i64)
1560 return Register();
1561 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1562 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1563 }
1564 return Register();
1565}
1566
1567Register fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Register Op0) {
1568 if ((Subtarget->hasVLX())) {
1569 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
1570 }
1571 if ((Subtarget->hasAVX())) {
1572 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
1573 }
1574 return Register();
1575}
1576
1577Register fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Register Op0) {
1578 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1579 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1580 }
1581 return Register();
1582}
1583
1584Register fastEmit_ISD_LRINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1585switch (RetVT.SimpleTy) {
1586 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Op0);
1587 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Op0);
1588 default: return Register();
1589}
1590}
1591
1592Register fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Register Op0) {
1593 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
1594}
1595
1596Register fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Register Op0) {
1597 if ((Subtarget->hasDQI())) {
1598 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1599 }
1600 return Register();
1601}
1602
1603Register fastEmit_ISD_LRINT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1604switch (RetVT.SimpleTy) {
1605 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Op0);
1606 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Op0);
1607 default: return Register();
1608}
1609}
1610
1611Register fastEmit_ISD_LRINT_r(MVT VT, MVT RetVT, Register Op0) {
1612 switch (VT.SimpleTy) {
1613 case MVT::f16: return fastEmit_ISD_LRINT_MVT_f16_r(RetVT, Op0);
1614 case MVT::f32: return fastEmit_ISD_LRINT_MVT_f32_r(RetVT, Op0);
1615 case MVT::f64: return fastEmit_ISD_LRINT_MVT_f64_r(RetVT, Op0);
1616 case MVT::v8f16: return fastEmit_ISD_LRINT_MVT_v8f16_r(RetVT, Op0);
1617 case MVT::v16f16: return fastEmit_ISD_LRINT_MVT_v16f16_r(RetVT, Op0);
1618 case MVT::v32f16: return fastEmit_ISD_LRINT_MVT_v32f16_r(RetVT, Op0);
1619 case MVT::v4f32: return fastEmit_ISD_LRINT_MVT_v4f32_r(RetVT, Op0);
1620 case MVT::v8f32: return fastEmit_ISD_LRINT_MVT_v8f32_r(RetVT, Op0);
1621 case MVT::v16f32: return fastEmit_ISD_LRINT_MVT_v16f32_r(RetVT, Op0);
1622 case MVT::v2f64: return fastEmit_ISD_LRINT_MVT_v2f64_r(RetVT, Op0);
1623 case MVT::v4f64: return fastEmit_ISD_LRINT_MVT_v4f64_r(RetVT, Op0);
1624 case MVT::v8f64: return fastEmit_ISD_LRINT_MVT_v8f64_r(RetVT, Op0);
1625 default: return Register();
1626 }
1627}
1628
1629// FastEmit functions for ISD::SCALAR_TO_VECTOR.
1630
1631Register fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
1632 if (RetVT.SimpleTy != MVT::v4i32)
1633 return Register();
1634 if ((Subtarget->hasAVX512())) {
1635 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIZrr, RC: &X86::VR128XRegClass, Op0);
1636 }
1637 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1638 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1639 }
1640 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1641 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1642 }
1643 return Register();
1644}
1645
1646Register fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
1647 if (RetVT.SimpleTy != MVT::v2i64)
1648 return Register();
1649 if ((Subtarget->hasAVX512())) {
1650 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIZrr, RC: &X86::VR128XRegClass, Op0);
1651 }
1652 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1653 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1654 }
1655 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1656 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1657 }
1658 return Register();
1659}
1660
1661Register fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
1662 switch (VT.SimpleTy) {
1663 case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0);
1664 case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0);
1665 default: return Register();
1666 }
1667}
1668
1669// FastEmit functions for ISD::SIGN_EXTEND.
1670
1671Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Register Op0) {
1672 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr8, RC: &X86::GR32RegClass, Op0);
1673}
1674
1675Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Register Op0) {
1676 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr8, RC: &X86::GR64RegClass, Op0);
1677}
1678
1679Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
1680switch (RetVT.SimpleTy) {
1681 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0);
1682 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0);
1683 default: return Register();
1684}
1685}
1686
1687Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Register Op0) {
1688 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr16, RC: &X86::GR32RegClass, Op0);
1689}
1690
1691Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Register Op0) {
1692 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr16, RC: &X86::GR64RegClass, Op0);
1693}
1694
1695Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, Register Op0) {
1696switch (RetVT.SimpleTy) {
1697 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0);
1698 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0);
1699 default: return Register();
1700}
1701}
1702
1703Register fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, Register Op0) {
1704 if (RetVT.SimpleTy != MVT::i64)
1705 return Register();
1706 if ((Subtarget->is64Bit())) {
1707 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr32, RC: &X86::GR64RegClass, Op0);
1708 }
1709 return Register();
1710}
1711
1712Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, Register Op0) {
1713 if (RetVT.SimpleTy != MVT::v2i64)
1714 return Register();
1715 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1716 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ128rk, RC: &X86::VR128XRegClass, Op0);
1717 }
1718 return Register();
1719}
1720
1721Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Register Op0) {
1722 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1723 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ128rk, RC: &X86::VR128XRegClass, Op0);
1724 }
1725 return Register();
1726}
1727
1728Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Register Op0) {
1729 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1730 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ256rk, RC: &X86::VR256XRegClass, Op0);
1731 }
1732 return Register();
1733}
1734
1735Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, Register Op0) {
1736switch (RetVT.SimpleTy) {
1737 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0);
1738 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0);
1739 default: return Register();
1740}
1741}
1742
1743Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Register Op0) {
1744 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1745 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ128rk, RC: &X86::VR128XRegClass, Op0);
1746 }
1747 return Register();
1748}
1749
1750Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Register Op0) {
1751 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1752 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ256rk, RC: &X86::VR256XRegClass, Op0);
1753 }
1754 return Register();
1755}
1756
1757Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Register Op0) {
1758 if ((Subtarget->hasDQI())) {
1759 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZrk, RC: &X86::VR512RegClass, Op0);
1760 }
1761 return Register();
1762}
1763
1764Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, Register Op0) {
1765switch (RetVT.SimpleTy) {
1766 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0);
1767 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0);
1768 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0);
1769 default: return Register();
1770}
1771}
1772
1773Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Register Op0) {
1774 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1775 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ128rk, RC: &X86::VR128XRegClass, Op0);
1776 }
1777 return Register();
1778}
1779
1780Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Register Op0) {
1781 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1782 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ256rk, RC: &X86::VR256XRegClass, Op0);
1783 }
1784 return Register();
1785}
1786
1787Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Register Op0) {
1788 if ((Subtarget->hasDQI())) {
1789 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZrk, RC: &X86::VR512RegClass, Op0);
1790 }
1791 return Register();
1792}
1793
1794Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, Register Op0) {
1795switch (RetVT.SimpleTy) {
1796 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0);
1797 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0);
1798 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0);
1799 default: return Register();
1800}
1801}
1802
1803Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Register Op0) {
1804 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1805 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ256rk, RC: &X86::VR256XRegClass, Op0);
1806 }
1807 return Register();
1808}
1809
1810Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Register Op0) {
1811 if ((Subtarget->hasBWI())) {
1812 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZrk, RC: &X86::VR512RegClass, Op0);
1813 }
1814 return Register();
1815}
1816
1817Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, Register Op0) {
1818switch (RetVT.SimpleTy) {
1819 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0);
1820 case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0);
1821 default: return Register();
1822}
1823}
1824
1825Register fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, Register Op0) {
1826 if (RetVT.SimpleTy != MVT::v64i8)
1827 return Register();
1828 if ((Subtarget->hasBWI())) {
1829 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZrk, RC: &X86::VR512RegClass, Op0);
1830 }
1831 return Register();
1832}
1833
1834Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Register Op0) {
1835 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1836 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWYrr, RC: &X86::VR256RegClass, Op0);
1837 }
1838 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1839 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
1840 }
1841 return Register();
1842}
1843
1844Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Register Op0) {
1845 if ((Subtarget->hasAVX512())) {
1846 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZrr, RC: &X86::VR512RegClass, Op0);
1847 }
1848 return Register();
1849}
1850
1851Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(MVT RetVT, Register Op0) {
1852switch (RetVT.SimpleTy) {
1853 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
1854 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
1855 default: return Register();
1856}
1857}
1858
1859Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(MVT RetVT, Register Op0) {
1860 if (RetVT.SimpleTy != MVT::v32i16)
1861 return Register();
1862 if ((Subtarget->hasBWI())) {
1863 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZrr, RC: &X86::VR512RegClass, Op0);
1864 }
1865 return Register();
1866}
1867
1868Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Register Op0) {
1869 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1870 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDYrr, RC: &X86::VR256RegClass, Op0);
1871 }
1872 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1873 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
1874 }
1875 return Register();
1876}
1877
1878Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Register Op0) {
1879 if ((Subtarget->hasAVX512())) {
1880 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZrr, RC: &X86::VR512RegClass, Op0);
1881 }
1882 return Register();
1883}
1884
1885Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(MVT RetVT, Register Op0) {
1886switch (RetVT.SimpleTy) {
1887 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
1888 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
1889 default: return Register();
1890}
1891}
1892
1893Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(MVT RetVT, Register Op0) {
1894 if (RetVT.SimpleTy != MVT::v16i32)
1895 return Register();
1896 if ((Subtarget->hasAVX512())) {
1897 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZrr, RC: &X86::VR512RegClass, Op0);
1898 }
1899 return Register();
1900}
1901
1902Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(MVT RetVT, Register Op0) {
1903 if (RetVT.SimpleTy != MVT::v4i64)
1904 return Register();
1905 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1906 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQYrr, RC: &X86::VR256RegClass, Op0);
1907 }
1908 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1909 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
1910 }
1911 return Register();
1912}
1913
1914Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(MVT RetVT, Register Op0) {
1915 if (RetVT.SimpleTy != MVT::v8i64)
1916 return Register();
1917 if ((Subtarget->hasAVX512())) {
1918 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZrr, RC: &X86::VR512RegClass, Op0);
1919 }
1920 return Register();
1921}
1922
1923Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
1924 switch (VT.SimpleTy) {
1925 case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0);
1926 case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0);
1927 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0);
1928 case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0);
1929 case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0);
1930 case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0);
1931 case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0);
1932 case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0);
1933 case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0);
1934 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(RetVT, Op0);
1935 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(RetVT, Op0);
1936 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(RetVT, Op0);
1937 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(RetVT, Op0);
1938 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(RetVT, Op0);
1939 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(RetVT, Op0);
1940 default: return Register();
1941 }
1942}
1943
1944// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
1945
1946Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1947 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1948 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1949 }
1950 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1951 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1952 }
1953 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1954 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
1955 }
1956 return Register();
1957}
1958
1959Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1960 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1961 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
1962 }
1963 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1964 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
1965 }
1966 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1967 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
1968 }
1969 return Register();
1970}
1971
1972Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Register Op0) {
1973 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1974 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDYrr, RC: &X86::VR256RegClass, Op0);
1975 }
1976 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1977 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
1978 }
1979 return Register();
1980}
1981
1982Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1983 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1984 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
1985 }
1986 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1987 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
1988 }
1989 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1990 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
1991 }
1992 return Register();
1993}
1994
1995Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Register Op0) {
1996 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1997 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQYrr, RC: &X86::VR256RegClass, Op0);
1998 }
1999 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2000 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
2001 }
2002 return Register();
2003}
2004
2005Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Register Op0) {
2006 if ((Subtarget->hasAVX512())) {
2007 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZrr, RC: &X86::VR512RegClass, Op0);
2008 }
2009 return Register();
2010}
2011
2012Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, Register Op0) {
2013switch (RetVT.SimpleTy) {
2014 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
2015 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
2016 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
2017 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
2018 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
2019 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
2020 default: return Register();
2021}
2022}
2023
2024Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Register Op0) {
2025 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2026 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
2027 }
2028 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2029 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
2030 }
2031 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2032 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
2033 }
2034 return Register();
2035}
2036
2037Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Register Op0) {
2038 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2039 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
2040 }
2041 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2042 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
2043 }
2044 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2045 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
2046 }
2047 return Register();
2048}
2049
2050Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Register Op0) {
2051 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2052 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQYrr, RC: &X86::VR256RegClass, Op0);
2053 }
2054 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2055 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
2056 }
2057 return Register();
2058}
2059
2060Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, Register Op0) {
2061switch (RetVT.SimpleTy) {
2062 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
2063 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
2064 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
2065 default: return Register();
2066}
2067}
2068
2069Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, Register Op0) {
2070 if (RetVT.SimpleTy != MVT::v2i64)
2071 return Register();
2072 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2073 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
2074 }
2075 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2076 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
2077 }
2078 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2079 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
2080 }
2081 return Register();
2082}
2083
2084Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, Register Op0) {
2085 switch (VT.SimpleTy) {
2086 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
2087 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
2088 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
2089 default: return Register();
2090 }
2091}
2092
2093// FastEmit functions for ISD::SINT_TO_FP.
2094
2095Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
2096 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2097 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2098 }
2099 return Register();
2100}
2101
2102Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
2103 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2104 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2105 }
2106 return Register();
2107}
2108
2109Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
2110switch (RetVT.SimpleTy) {
2111 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2112 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2113 default: return Register();
2114}
2115}
2116
2117Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
2118 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2119 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2120 }
2121 return Register();
2122}
2123
2124Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
2125 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2126 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2127 }
2128 return Register();
2129}
2130
2131Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
2132switch (RetVT.SimpleTy) {
2133 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2134 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2135 default: return Register();
2136}
2137}
2138
2139Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2140 if (RetVT.SimpleTy != MVT::v8f16)
2141 return Register();
2142 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2144 }
2145 return Register();
2146}
2147
2148Register fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
2149 if (RetVT.SimpleTy != MVT::v16f16)
2150 return Register();
2151 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2152 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2153 }
2154 return Register();
2155}
2156
2157Register fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
2158 if (RetVT.SimpleTy != MVT::v32f16)
2159 return Register();
2160 if ((Subtarget->hasFP16())) {
2161 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2162 }
2163 return Register();
2164}
2165
2166Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
2167 if ((Subtarget->hasVLX())) {
2168 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2169 }
2170 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2171 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2172 }
2173 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2174 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2175 }
2176 return Register();
2177}
2178
2179Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
2180 if ((Subtarget->hasVLX())) {
2181 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2182 }
2183 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2184 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2185 }
2186 return Register();
2187}
2188
2189Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2190switch (RetVT.SimpleTy) {
2191 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2192 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2193 default: return Register();
2194}
2195}
2196
2197Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
2198 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2199 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2200 }
2201 return Register();
2202}
2203
2204Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
2205 if ((Subtarget->hasVLX())) {
2206 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2207 }
2208 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2210 }
2211 return Register();
2212}
2213
2214Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
2215 if ((Subtarget->hasAVX512())) {
2216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2217 }
2218 return Register();
2219}
2220
2221Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
2222switch (RetVT.SimpleTy) {
2223 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2224 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2225 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2226 default: return Register();
2227}
2228}
2229
2230Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
2231 if ((Subtarget->hasFP16())) {
2232 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2233 }
2234 return Register();
2235}
2236
2237Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
2238 if ((Subtarget->hasAVX512())) {
2239 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2240 }
2241 return Register();
2242}
2243
2244Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
2245switch (RetVT.SimpleTy) {
2246 case MVT::v16f16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2247 case MVT::v16f32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2248 default: return Register();
2249}
2250}
2251
2252Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2253 if (RetVT.SimpleTy != MVT::v2f64)
2254 return Register();
2255 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2256 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2257 }
2258 return Register();
2259}
2260
2261Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
2262 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2263 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2264 }
2265 return Register();
2266}
2267
2268Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
2269 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2270 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2271 }
2272 return Register();
2273}
2274
2275Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
2276switch (RetVT.SimpleTy) {
2277 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2278 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2279 default: return Register();
2280}
2281}
2282
2283Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
2284 if ((Subtarget->hasFP16())) {
2285 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2286 }
2287 return Register();
2288}
2289
2290Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
2291 if ((Subtarget->hasDQI())) {
2292 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2293 }
2294 return Register();
2295}
2296
2297Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
2298 if ((Subtarget->hasDQI())) {
2299 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2300 }
2301 return Register();
2302}
2303
2304Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
2305switch (RetVT.SimpleTy) {
2306 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
2307 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
2308 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
2309 default: return Register();
2310}
2311}
2312
2313Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
2314 switch (VT.SimpleTy) {
2315 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
2316 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
2317 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2318 case MVT::v16i16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
2319 case MVT::v32i16: return fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
2320 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2321 case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
2322 case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
2323 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
2324 case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
2325 case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
2326 default: return Register();
2327 }
2328}
2329
2330// FastEmit functions for ISD::STRICT_FP_EXTEND.
2331
2332Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2333 if (RetVT.SimpleTy != MVT::f64)
2334 return Register();
2335 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2336 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
2337 }
2338 return Register();
2339}
2340
2341Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Register Op0) {
2342 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2343 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
2344 }
2345 return Register();
2346}
2347
2348Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Register Op0) {
2349 if ((Subtarget->hasFP16())) {
2350 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
2351 }
2352 return Register();
2353}
2354
2355Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2356switch (RetVT.SimpleTy) {
2357 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
2358 case MVT::v8f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
2359 default: return Register();
2360}
2361}
2362
2363Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(MVT RetVT, Register Op0) {
2364 if (RetVT.SimpleTy != MVT::v16f32)
2365 return Register();
2366 if ((Subtarget->hasFP16())) {
2367 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
2368 }
2369 return Register();
2370}
2371
2372Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2373 if (RetVT.SimpleTy != MVT::v4f64)
2374 return Register();
2375 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2376 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2377 }
2378 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2379 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
2380 }
2381 return Register();
2382}
2383
2384Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(MVT RetVT, Register Op0) {
2385 if (RetVT.SimpleTy != MVT::v8f64)
2386 return Register();
2387 if ((Subtarget->hasAVX512())) {
2388 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
2389 }
2390 return Register();
2391}
2392
2393Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2394 switch (VT.SimpleTy) {
2395 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2396 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
2397 case MVT::v16f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
2398 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
2399 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
2400 default: return Register();
2401 }
2402}
2403
2404// FastEmit functions for ISD::STRICT_FP_ROUND.
2405
2406Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2407 if (RetVT.SimpleTy != MVT::f32)
2408 return Register();
2409 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2410 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
2411 }
2412 return Register();
2413}
2414
2415Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2416 switch (VT.SimpleTy) {
2417 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
2418 default: return Register();
2419 }
2420}
2421
2422// FastEmit functions for ISD::STRICT_FP_TO_SINT.
2423
2424Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
2425 if ((Subtarget->hasFP16())) {
2426 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
2427 }
2428 return Register();
2429}
2430
2431Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
2432 if ((Subtarget->hasFP16())) {
2433 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2434 }
2435 return Register();
2436}
2437
2438Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
2439switch (RetVT.SimpleTy) {
2440 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
2441 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
2442 default: return Register();
2443}
2444}
2445
2446Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
2447 if ((Subtarget->hasAVX512())) {
2448 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
2449 }
2450 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2451 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2452 }
2453 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2455 }
2456 return Register();
2457}
2458
2459Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
2460 if ((Subtarget->hasAVX512())) {
2461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2462 }
2463 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2464 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2465 }
2466 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2467 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2468 }
2469 return Register();
2470}
2471
2472Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
2473switch (RetVT.SimpleTy) {
2474 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
2475 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
2476 default: return Register();
2477}
2478}
2479
2480Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
2481 if ((Subtarget->hasAVX512())) {
2482 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
2483 }
2484 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2485 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2486 }
2487 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2488 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2489 }
2490 return Register();
2491}
2492
2493Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
2494 if ((Subtarget->hasAVX512())) {
2495 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2496 }
2497 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2498 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2499 }
2500 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2502 }
2503 return Register();
2504}
2505
2506Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
2507switch (RetVT.SimpleTy) {
2508 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
2509 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
2510 default: return Register();
2511}
2512}
2513
2514Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
2515 if (RetVT.SimpleTy != MVT::v4i32)
2516 return Register();
2517 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2518 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
2519 }
2520 return Register();
2521}
2522
2523Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2524 switch (VT.SimpleTy) {
2525 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
2526 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
2527 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
2528 case MVT::v4f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
2529 default: return Register();
2530 }
2531}
2532
2533// FastEmit functions for ISD::STRICT_FP_TO_UINT.
2534
2535Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
2536 if ((Subtarget->hasFP16())) {
2537 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
2538 }
2539 return Register();
2540}
2541
2542Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
2543 if ((Subtarget->hasFP16())) {
2544 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2545 }
2546 return Register();
2547}
2548
2549Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
2550switch (RetVT.SimpleTy) {
2551 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
2552 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
2553 default: return Register();
2554}
2555}
2556
2557Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
2558 if ((Subtarget->hasAVX512())) {
2559 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
2560 }
2561 return Register();
2562}
2563
2564Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
2565 if ((Subtarget->hasAVX512())) {
2566 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2567 }
2568 return Register();
2569}
2570
2571Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
2572switch (RetVT.SimpleTy) {
2573 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
2574 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
2575 default: return Register();
2576}
2577}
2578
2579Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
2580 if ((Subtarget->hasAVX512())) {
2581 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
2582 }
2583 return Register();
2584}
2585
2586Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
2587 if ((Subtarget->hasAVX512())) {
2588 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2589 }
2590 return Register();
2591}
2592
2593Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
2594switch (RetVT.SimpleTy) {
2595 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
2596 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
2597 default: return Register();
2598}
2599}
2600
2601Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2602 switch (VT.SimpleTy) {
2603 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
2604 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
2605 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
2606 default: return Register();
2607 }
2608}
2609
2610// FastEmit functions for ISD::STRICT_FSQRT.
2611
2612Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
2613 if (RetVT.SimpleTy != MVT::f32)
2614 return Register();
2615 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2616 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
2617 }
2618 if ((!Subtarget->hasSSE1())) {
2619 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
2620 }
2621 return Register();
2622}
2623
2624Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
2625 if (RetVT.SimpleTy != MVT::f64)
2626 return Register();
2627 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2628 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
2629 }
2630 if ((!Subtarget->hasSSE2())) {
2631 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
2632 }
2633 return Register();
2634}
2635
2636Register fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(MVT RetVT, Register Op0) {
2637 if (RetVT.SimpleTy != MVT::f80)
2638 return Register();
2639 if ((Subtarget->hasX87())) {
2640 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
2641 }
2642 return Register();
2643}
2644
2645Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2646 if (RetVT.SimpleTy != MVT::v8f16)
2647 return Register();
2648 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2649 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
2650 }
2651 return Register();
2652}
2653
2654Register fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(MVT RetVT, Register Op0) {
2655 if (RetVT.SimpleTy != MVT::v16f16)
2656 return Register();
2657 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2658 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
2659 }
2660 return Register();
2661}
2662
2663Register fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(MVT RetVT, Register Op0) {
2664 if (RetVT.SimpleTy != MVT::v32f16)
2665 return Register();
2666 if ((Subtarget->hasFP16())) {
2667 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
2668 }
2669 return Register();
2670}
2671
2672Register fastEmit_ISD_STRICT_FSQRT_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2673 if (RetVT.SimpleTy != MVT::v8bf16)
2674 return Register();
2675 if ((Subtarget->hasAVX10_2())) {
2676 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
2677 }
2678 return Register();
2679}
2680
2681Register fastEmit_ISD_STRICT_FSQRT_MVT_v16bf16_r(MVT RetVT, Register Op0) {
2682 if (RetVT.SimpleTy != MVT::v16bf16)
2683 return Register();
2684 if ((Subtarget->hasAVX10_2())) {
2685 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
2686 }
2687 return Register();
2688}
2689
2690Register fastEmit_ISD_STRICT_FSQRT_MVT_v32bf16_r(MVT RetVT, Register Op0) {
2691 if (RetVT.SimpleTy != MVT::v32bf16)
2692 return Register();
2693 if ((Subtarget->hasAVX10_2_512())) {
2694 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
2695 }
2696 return Register();
2697}
2698
2699Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2700 if (RetVT.SimpleTy != MVT::v4f32)
2701 return Register();
2702 if ((Subtarget->hasVLX())) {
2703 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
2704 }
2705 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2706 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
2707 }
2708 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2709 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
2710 }
2711 return Register();
2712}
2713
2714Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
2715 if (RetVT.SimpleTy != MVT::v8f32)
2716 return Register();
2717 if ((Subtarget->hasVLX())) {
2718 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
2719 }
2720 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2721 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
2722 }
2723 return Register();
2724}
2725
2726Register fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(MVT RetVT, Register Op0) {
2727 if (RetVT.SimpleTy != MVT::v16f32)
2728 return Register();
2729 if ((Subtarget->hasAVX512())) {
2730 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
2731 }
2732 return Register();
2733}
2734
2735Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2736 if (RetVT.SimpleTy != MVT::v2f64)
2737 return Register();
2738 if ((Subtarget->hasVLX())) {
2739 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
2740 }
2741 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2742 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
2743 }
2744 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2745 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
2746 }
2747 return Register();
2748}
2749
2750Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(MVT RetVT, Register Op0) {
2751 if (RetVT.SimpleTy != MVT::v4f64)
2752 return Register();
2753 if ((Subtarget->hasVLX())) {
2754 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
2755 }
2756 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2757 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
2758 }
2759 return Register();
2760}
2761
2762Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(MVT RetVT, Register Op0) {
2763 if (RetVT.SimpleTy != MVT::v8f64)
2764 return Register();
2765 if ((Subtarget->hasAVX512())) {
2766 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
2767 }
2768 return Register();
2769}
2770
2771Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
2772 switch (VT.SimpleTy) {
2773 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
2774 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
2775 case MVT::f80: return fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(RetVT, Op0);
2776 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
2777 case MVT::v16f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(RetVT, Op0);
2778 case MVT::v32f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(RetVT, Op0);
2779 case MVT::v8bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8bf16_r(RetVT, Op0);
2780 case MVT::v16bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16bf16_r(RetVT, Op0);
2781 case MVT::v32bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32bf16_r(RetVT, Op0);
2782 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
2783 case MVT::v8f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(RetVT, Op0);
2784 case MVT::v16f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(RetVT, Op0);
2785 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
2786 case MVT::v4f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(RetVT, Op0);
2787 case MVT::v8f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(RetVT, Op0);
2788 default: return Register();
2789 }
2790}
2791
2792// FastEmit functions for ISD::STRICT_SINT_TO_FP.
2793
2794Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
2795 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2796 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2797 }
2798 return Register();
2799}
2800
2801Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
2802 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2803 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2804 }
2805 return Register();
2806}
2807
2808Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
2809switch (RetVT.SimpleTy) {
2810 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2811 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2812 default: return Register();
2813}
2814}
2815
2816Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
2817 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2818 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2819 }
2820 return Register();
2821}
2822
2823Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
2824 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2825 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2826 }
2827 return Register();
2828}
2829
2830Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
2831switch (RetVT.SimpleTy) {
2832 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2833 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2834 default: return Register();
2835}
2836}
2837
2838Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2839 if (RetVT.SimpleTy != MVT::v8f16)
2840 return Register();
2841 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2842 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2843 }
2844 return Register();
2845}
2846
2847Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
2848 if (RetVT.SimpleTy != MVT::v16f16)
2849 return Register();
2850 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2851 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2852 }
2853 return Register();
2854}
2855
2856Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
2857 if (RetVT.SimpleTy != MVT::v32f16)
2858 return Register();
2859 if ((Subtarget->hasFP16())) {
2860 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2861 }
2862 return Register();
2863}
2864
2865Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
2866 if ((Subtarget->hasVLX())) {
2867 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2868 }
2869 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2870 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2871 }
2872 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2873 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2874 }
2875 return Register();
2876}
2877
2878Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
2879 if ((Subtarget->hasVLX())) {
2880 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2881 }
2882 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2883 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2884 }
2885 return Register();
2886}
2887
2888Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2889switch (RetVT.SimpleTy) {
2890 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2891 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2892 default: return Register();
2893}
2894}
2895
2896Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
2897 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2898 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2899 }
2900 return Register();
2901}
2902
2903Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
2904 if ((Subtarget->hasVLX())) {
2905 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2906 }
2907 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2908 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2909 }
2910 return Register();
2911}
2912
2913Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
2914 if ((Subtarget->hasAVX512())) {
2915 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2916 }
2917 return Register();
2918}
2919
2920Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
2921switch (RetVT.SimpleTy) {
2922 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2923 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2924 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2925 default: return Register();
2926}
2927}
2928
2929Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
2930 if ((Subtarget->hasFP16())) {
2931 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2932 }
2933 return Register();
2934}
2935
2936Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
2937 if ((Subtarget->hasAVX512())) {
2938 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2939 }
2940 return Register();
2941}
2942
2943Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
2944switch (RetVT.SimpleTy) {
2945 case MVT::v16f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2946 case MVT::v16f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2947 default: return Register();
2948}
2949}
2950
2951Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2952 if (RetVT.SimpleTy != MVT::v2f64)
2953 return Register();
2954 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2955 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2956 }
2957 return Register();
2958}
2959
2960Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
2961 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2963 }
2964 return Register();
2965}
2966
2967Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
2968 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2969 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2970 }
2971 return Register();
2972}
2973
2974Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
2975switch (RetVT.SimpleTy) {
2976 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2977 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2978 default: return Register();
2979}
2980}
2981
2982Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
2983 if ((Subtarget->hasFP16())) {
2984 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2985 }
2986 return Register();
2987}
2988
2989Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
2990 if ((Subtarget->hasDQI())) {
2991 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2992 }
2993 return Register();
2994}
2995
2996Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
2997 if ((Subtarget->hasDQI())) {
2998 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2999 }
3000 return Register();
3001}
3002
3003Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3004switch (RetVT.SimpleTy) {
3005 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3006 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3007 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3008 default: return Register();
3009}
3010}
3011
3012Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3013 switch (VT.SimpleTy) {
3014 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
3015 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
3016 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3017 case MVT::v16i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3018 case MVT::v32i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3019 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3020 case MVT::v8i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3021 case MVT::v16i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3022 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3023 case MVT::v4i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3024 case MVT::v8i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3025 default: return Register();
3026 }
3027}
3028
3029// FastEmit functions for ISD::STRICT_UINT_TO_FP.
3030
3031Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3032 if (RetVT.SimpleTy != MVT::v8f16)
3033 return Register();
3034 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3035 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3036 }
3037 return Register();
3038}
3039
3040Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
3041 if (RetVT.SimpleTy != MVT::v16f16)
3042 return Register();
3043 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3044 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3045 }
3046 return Register();
3047}
3048
3049Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
3050 if (RetVT.SimpleTy != MVT::v32f16)
3051 return Register();
3052 if ((Subtarget->hasFP16())) {
3053 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3054 }
3055 return Register();
3056}
3057
3058Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
3059 if ((Subtarget->hasVLX())) {
3060 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3061 }
3062 return Register();
3063}
3064
3065Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
3066 if ((Subtarget->hasVLX())) {
3067 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3068 }
3069 return Register();
3070}
3071
3072Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3073switch (RetVT.SimpleTy) {
3074 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3075 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3076 default: return Register();
3077}
3078}
3079
3080Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
3081 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3082 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3083 }
3084 return Register();
3085}
3086
3087Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
3088 if ((Subtarget->hasVLX())) {
3089 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3090 }
3091 return Register();
3092}
3093
3094Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
3095 if ((Subtarget->hasAVX512())) {
3096 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3097 }
3098 return Register();
3099}
3100
3101Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
3102switch (RetVT.SimpleTy) {
3103 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3104 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3105 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3106 default: return Register();
3107}
3108}
3109
3110Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
3111 if ((Subtarget->hasFP16())) {
3112 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3113 }
3114 return Register();
3115}
3116
3117Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
3118 if ((Subtarget->hasAVX512())) {
3119 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3120 }
3121 return Register();
3122}
3123
3124Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
3125switch (RetVT.SimpleTy) {
3126 case MVT::v16f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3127 case MVT::v16f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3128 default: return Register();
3129}
3130}
3131
3132Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
3133 if (RetVT.SimpleTy != MVT::v2f64)
3134 return Register();
3135 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3136 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3137 }
3138 return Register();
3139}
3140
3141Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
3142 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3144 }
3145 return Register();
3146}
3147
3148Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3149 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3150 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3151 }
3152 return Register();
3153}
3154
3155Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3156switch (RetVT.SimpleTy) {
3157 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3158 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3159 default: return Register();
3160}
3161}
3162
3163Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3164 if ((Subtarget->hasFP16())) {
3165 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3166 }
3167 return Register();
3168}
3169
3170Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3171 if ((Subtarget->hasDQI())) {
3172 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3173 }
3174 return Register();
3175}
3176
3177Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3178 if ((Subtarget->hasDQI())) {
3179 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3180 }
3181 return Register();
3182}
3183
3184Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3185switch (RetVT.SimpleTy) {
3186 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3187 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3188 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3189 default: return Register();
3190}
3191}
3192
3193Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3194 switch (VT.SimpleTy) {
3195 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3196 case MVT::v16i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3197 case MVT::v32i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3198 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3199 case MVT::v8i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3200 case MVT::v16i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3201 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3202 case MVT::v4i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3203 case MVT::v8i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3204 default: return Register();
3205 }
3206}
3207
3208// FastEmit functions for ISD::TRUNCATE.
3209
3210Register fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, Register Op0) {
3211 if (RetVT.SimpleTy != MVT::i8)
3212 return Register();
3213 if ((Subtarget->is64Bit())) {
3214 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: X86::sub_8bit);
3215 }
3216 return Register();
3217}
3218
3219Register fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Register Op0) {
3220 if ((Subtarget->is64Bit())) {
3221 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3222 }
3223 return Register();
3224}
3225
3226Register fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Register Op0) {
3227 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3228}
3229
3230Register fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, Register Op0) {
3231switch (RetVT.SimpleTy) {
3232 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0);
3233 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0);
3234 default: return Register();
3235}
3236}
3237
3238Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Register Op0) {
3239 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3240}
3241
3242Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Register Op0) {
3243 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3244}
3245
3246Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Register Op0) {
3247 return fastEmitInst_extractsubreg(RetVT: MVT::i32, Op0, Idx: X86::sub_32bit);
3248}
3249
3250Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
3251switch (RetVT.SimpleTy) {
3252 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0);
3253 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0);
3254 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0);
3255 default: return Register();
3256}
3257}
3258
3259Register fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, Register Op0) {
3260 if (RetVT.SimpleTy != MVT::v16i8)
3261 return Register();
3262 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3263 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ256rr, RC: &X86::VR128XRegClass, Op0);
3264 }
3265 return Register();
3266}
3267
3268Register fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, Register Op0) {
3269 if (RetVT.SimpleTy != MVT::v32i8)
3270 return Register();
3271 if ((Subtarget->hasBWI())) {
3272 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZrr, RC: &X86::VR256XRegClass, Op0);
3273 }
3274 return Register();
3275}
3276
3277Register fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, Register Op0) {
3278 if (RetVT.SimpleTy != MVT::v8i16)
3279 return Register();
3280 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3281 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ256rr, RC: &X86::VR128XRegClass, Op0);
3282 }
3283 return Register();
3284}
3285
3286Register fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Register Op0) {
3287 if ((Subtarget->hasAVX512())) {
3288 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZrr, RC: &X86::VR128XRegClass, Op0);
3289 }
3290 return Register();
3291}
3292
3293Register fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Register Op0) {
3294 if ((Subtarget->hasAVX512())) {
3295 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZrr, RC: &X86::VR256XRegClass, Op0);
3296 }
3297 return Register();
3298}
3299
3300Register fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, Register Op0) {
3301switch (RetVT.SimpleTy) {
3302 case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0);
3303 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0);
3304 default: return Register();
3305}
3306}
3307
3308Register fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, Register Op0) {
3309 if (RetVT.SimpleTy != MVT::v4i32)
3310 return Register();
3311 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3312 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ256rr, RC: &X86::VR128XRegClass, Op0);
3313 }
3314 return Register();
3315}
3316
3317Register fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Register Op0) {
3318 if ((Subtarget->hasAVX512())) {
3319 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZrr, RC: &X86::VR128XRegClass, Op0);
3320 }
3321 return Register();
3322}
3323
3324Register fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Register Op0) {
3325 if ((Subtarget->hasAVX512())) {
3326 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZrr, RC: &X86::VR256XRegClass, Op0);
3327 }
3328 return Register();
3329}
3330
3331Register fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, Register Op0) {
3332switch (RetVT.SimpleTy) {
3333 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0);
3334 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0);
3335 default: return Register();
3336}
3337}
3338
3339Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3340 switch (VT.SimpleTy) {
3341 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0);
3342 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0);
3343 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
3344 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0);
3345 case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0);
3346 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0);
3347 case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0);
3348 case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0);
3349 case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0);
3350 default: return Register();
3351 }
3352}
3353
3354// FastEmit functions for ISD::UINT_TO_FP.
3355
3356Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3357 if (RetVT.SimpleTy != MVT::v8f16)
3358 return Register();
3359 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3360 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3361 }
3362 return Register();
3363}
3364
3365Register fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
3366 if (RetVT.SimpleTy != MVT::v16f16)
3367 return Register();
3368 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3369 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3370 }
3371 return Register();
3372}
3373
3374Register fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
3375 if (RetVT.SimpleTy != MVT::v32f16)
3376 return Register();
3377 if ((Subtarget->hasFP16())) {
3378 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3379 }
3380 return Register();
3381}
3382
3383Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
3384 if ((Subtarget->hasVLX())) {
3385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3386 }
3387 return Register();
3388}
3389
3390Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
3391 if ((Subtarget->hasVLX())) {
3392 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3393 }
3394 return Register();
3395}
3396
3397Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3398switch (RetVT.SimpleTy) {
3399 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3400 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3401 default: return Register();
3402}
3403}
3404
3405Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
3406 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3407 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3408 }
3409 return Register();
3410}
3411
3412Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
3413 if ((Subtarget->hasVLX())) {
3414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3415 }
3416 return Register();
3417}
3418
3419Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
3420 if ((Subtarget->hasAVX512())) {
3421 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3422 }
3423 return Register();
3424}
3425
3426Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
3427switch (RetVT.SimpleTy) {
3428 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3429 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3430 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3431 default: return Register();
3432}
3433}
3434
3435Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
3436 if ((Subtarget->hasFP16())) {
3437 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3438 }
3439 return Register();
3440}
3441
3442Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
3443 if ((Subtarget->hasAVX512())) {
3444 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3445 }
3446 return Register();
3447}
3448
3449Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
3450switch (RetVT.SimpleTy) {
3451 case MVT::v16f16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3452 case MVT::v16f32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3453 default: return Register();
3454}
3455}
3456
3457Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
3458 if (RetVT.SimpleTy != MVT::v2f64)
3459 return Register();
3460 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3462 }
3463 return Register();
3464}
3465
3466Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
3467 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3469 }
3470 return Register();
3471}
3472
3473Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3474 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3475 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3476 }
3477 return Register();
3478}
3479
3480Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3481switch (RetVT.SimpleTy) {
3482 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3483 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3484 default: return Register();
3485}
3486}
3487
3488Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3489 if ((Subtarget->hasFP16())) {
3490 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3491 }
3492 return Register();
3493}
3494
3495Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3496 if ((Subtarget->hasDQI())) {
3497 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3498 }
3499 return Register();
3500}
3501
3502Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3503 if ((Subtarget->hasDQI())) {
3504 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3505 }
3506 return Register();
3507}
3508
3509Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3510switch (RetVT.SimpleTy) {
3511 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3512 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3513 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3514 default: return Register();
3515}
3516}
3517
3518Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3519 switch (VT.SimpleTy) {
3520 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3521 case MVT::v16i16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3522 case MVT::v32i16: return fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3523 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3524 case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3525 case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3526 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3527 case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3528 case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3529 default: return Register();
3530 }
3531}
3532
3533// FastEmit functions for ISD::ZERO_EXTEND.
3534
3535Register fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
3536 if (RetVT.SimpleTy != MVT::i32)
3537 return Register();
3538 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
3539}
3540
3541Register fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, Register Op0) {
3542 if (RetVT.SimpleTy != MVT::i32)
3543 return Register();
3544 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr16, RC: &X86::GR32RegClass, Op0);
3545}
3546
3547Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Register Op0) {
3548 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3549 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWYrr, RC: &X86::VR256RegClass, Op0);
3550 }
3551 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3552 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
3553 }
3554 return Register();
3555}
3556
3557Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Register Op0) {
3558 if ((Subtarget->hasAVX512())) {
3559 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZrr, RC: &X86::VR512RegClass, Op0);
3560 }
3561 return Register();
3562}
3563
3564Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(MVT RetVT, Register Op0) {
3565switch (RetVT.SimpleTy) {
3566 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
3567 case MVT::v16i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
3568 default: return Register();
3569}
3570}
3571
3572Register fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(MVT RetVT, Register Op0) {
3573 if (RetVT.SimpleTy != MVT::v32i16)
3574 return Register();
3575 if ((Subtarget->hasBWI())) {
3576 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZrr, RC: &X86::VR512RegClass, Op0);
3577 }
3578 return Register();
3579}
3580
3581Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Register Op0) {
3582 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3583 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDYrr, RC: &X86::VR256RegClass, Op0);
3584 }
3585 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3586 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
3587 }
3588 return Register();
3589}
3590
3591Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Register Op0) {
3592 if ((Subtarget->hasAVX512())) {
3593 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZrr, RC: &X86::VR512RegClass, Op0);
3594 }
3595 return Register();
3596}
3597
3598Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(MVT RetVT, Register Op0) {
3599switch (RetVT.SimpleTy) {
3600 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
3601 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
3602 default: return Register();
3603}
3604}
3605
3606Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(MVT RetVT, Register Op0) {
3607 if (RetVT.SimpleTy != MVT::v16i32)
3608 return Register();
3609 if ((Subtarget->hasAVX512())) {
3610 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZrr, RC: &X86::VR512RegClass, Op0);
3611 }
3612 return Register();
3613}
3614
3615Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(MVT RetVT, Register Op0) {
3616 if (RetVT.SimpleTy != MVT::v4i64)
3617 return Register();
3618 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3619 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQYrr, RC: &X86::VR256RegClass, Op0);
3620 }
3621 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3622 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
3623 }
3624 return Register();
3625}
3626
3627Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(MVT RetVT, Register Op0) {
3628 if (RetVT.SimpleTy != MVT::v8i64)
3629 return Register();
3630 if ((Subtarget->hasAVX512())) {
3631 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZrr, RC: &X86::VR512RegClass, Op0);
3632 }
3633 return Register();
3634}
3635
3636Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3637 switch (VT.SimpleTy) {
3638 case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0);
3639 case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0);
3640 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(RetVT, Op0);
3641 case MVT::v32i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(RetVT, Op0);
3642 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(RetVT, Op0);
3643 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(RetVT, Op0);
3644 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(RetVT, Op0);
3645 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(RetVT, Op0);
3646 default: return Register();
3647 }
3648}
3649
3650// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
3651
3652Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Register Op0) {
3653 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3654 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3655 }
3656 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3657 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3658 }
3659 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3660 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
3661 }
3662 return Register();
3663}
3664
3665Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Register Op0) {
3666 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3667 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3668 }
3669 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3670 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3671 }
3672 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3673 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
3674 }
3675 return Register();
3676}
3677
3678Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Register Op0) {
3679 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3680 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDYrr, RC: &X86::VR256RegClass, Op0);
3681 }
3682 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3683 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
3684 }
3685 return Register();
3686}
3687
3688Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Register Op0) {
3689 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3690 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3691 }
3692 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3693 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3694 }
3695 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3696 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
3697 }
3698 return Register();
3699}
3700
3701Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Register Op0) {
3702 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3703 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQYrr, RC: &X86::VR256RegClass, Op0);
3704 }
3705 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3706 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
3707 }
3708 return Register();
3709}
3710
3711Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Register Op0) {
3712 if ((Subtarget->hasAVX512())) {
3713 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZrr, RC: &X86::VR512RegClass, Op0);
3714 }
3715 return Register();
3716}
3717
3718Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, Register Op0) {
3719switch (RetVT.SimpleTy) {
3720 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
3721 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
3722 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
3723 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
3724 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
3725 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
3726 default: return Register();
3727}
3728}
3729
3730Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Register Op0) {
3731 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3732 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3733 }
3734 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3735 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3736 }
3737 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3738 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
3739 }
3740 return Register();
3741}
3742
3743Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Register Op0) {
3744 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3745 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3746 }
3747 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3748 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3749 }
3750 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3751 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
3752 }
3753 return Register();
3754}
3755
3756Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Register Op0) {
3757 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3758 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQYrr, RC: &X86::VR256RegClass, Op0);
3759 }
3760 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3761 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
3762 }
3763 return Register();
3764}
3765
3766Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, Register Op0) {
3767switch (RetVT.SimpleTy) {
3768 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
3769 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
3770 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
3771 default: return Register();
3772}
3773}
3774
3775Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, Register Op0) {
3776 if (RetVT.SimpleTy != MVT::v2i64)
3777 return Register();
3778 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3779 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3780 }
3781 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3782 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3783 }
3784 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3785 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
3786 }
3787 return Register();
3788}
3789
3790Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, Register Op0) {
3791 switch (VT.SimpleTy) {
3792 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
3793 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
3794 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
3795 default: return Register();
3796 }
3797}
3798
3799// FastEmit functions for X86ISD::CALL.
3800
3801Register fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, Register Op0) {
3802 if (RetVT.SimpleTy != MVT::isVoid)
3803 return Register();
3804 if ((!Subtarget->is64Bit())) {
3805 return fastEmitInst_r(MachineInstOpcode: X86::CALL16r, RC: &X86::GR16RegClass, Op0);
3806 }
3807 return Register();
3808}
3809
3810Register fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
3811 if (RetVT.SimpleTy != MVT::isVoid)
3812 return Register();
3813 if ((!Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3814 return fastEmitInst_r(MachineInstOpcode: X86::INDIRECT_THUNK_CALL32, RC: &X86::GR32RegClass, Op0);
3815 }
3816 if ((!Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3817 return fastEmitInst_r(MachineInstOpcode: X86::CALL32r, RC: &X86::GR32RegClass, Op0);
3818 }
3819 return Register();
3820}
3821
3822Register fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
3823 if (RetVT.SimpleTy != MVT::isVoid)
3824 return Register();
3825 if ((MF->getFunction().getParent()->getModuleFlag(Key: "import-call-optimization")) && (Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3826 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r_ImpCall, RC: &X86::GR64_ARegClass, Op0);
3827 }
3828 if ((Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3829 return fastEmitInst_r(MachineInstOpcode: X86::INDIRECT_THUNK_CALL64, RC: &X86::GR64RegClass, Op0);
3830 }
3831 if ((!MF->getFunction().getParent()->getModuleFlag(Key: "import-call-optimization")) && (Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3832 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r, RC: &X86::GR64RegClass, Op0);
3833 }
3834 return Register();
3835}
3836
3837Register fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
3838 switch (VT.SimpleTy) {
3839 case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0);
3840 case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0);
3841 case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0);
3842 default: return Register();
3843 }
3844}
3845
3846// FastEmit functions for X86ISD::CONFLICT.
3847
3848Register fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, Register Op0) {
3849 if (RetVT.SimpleTy != MVT::v4i32)
3850 return Register();
3851 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3852 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ128rr, RC: &X86::VR128XRegClass, Op0);
3853 }
3854 return Register();
3855}
3856
3857Register fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, Register Op0) {
3858 if (RetVT.SimpleTy != MVT::v8i32)
3859 return Register();
3860 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3861 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ256rr, RC: &X86::VR256XRegClass, Op0);
3862 }
3863 return Register();
3864}
3865
3866Register fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, Register Op0) {
3867 if (RetVT.SimpleTy != MVT::v16i32)
3868 return Register();
3869 if ((Subtarget->hasCDI())) {
3870 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZrr, RC: &X86::VR512RegClass, Op0);
3871 }
3872 return Register();
3873}
3874
3875Register fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, Register Op0) {
3876 if (RetVT.SimpleTy != MVT::v2i64)
3877 return Register();
3878 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3879 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ128rr, RC: &X86::VR128XRegClass, Op0);
3880 }
3881 return Register();
3882}
3883
3884Register fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, Register Op0) {
3885 if (RetVT.SimpleTy != MVT::v4i64)
3886 return Register();
3887 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3888 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ256rr, RC: &X86::VR256XRegClass, Op0);
3889 }
3890 return Register();
3891}
3892
3893Register fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, Register Op0) {
3894 if (RetVT.SimpleTy != MVT::v8i64)
3895 return Register();
3896 if ((Subtarget->hasCDI())) {
3897 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZrr, RC: &X86::VR512RegClass, Op0);
3898 }
3899 return Register();
3900}
3901
3902Register fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, Register Op0) {
3903 switch (VT.SimpleTy) {
3904 case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0);
3905 case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0);
3906 case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0);
3907 case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0);
3908 case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0);
3909 case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0);
3910 default: return Register();
3911 }
3912}
3913
3914// FastEmit functions for X86ISD::CVTNEPS2BF16.
3915
3916Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(MVT RetVT, Register Op0) {
3917 if (RetVT.SimpleTy != MVT::v8bf16)
3918 return Register();
3919 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3920 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0);
3921 }
3922 if ((Subtarget->hasAVXNECONVERT())) {
3923 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16rr, RC: &X86::VR128RegClass, Op0);
3924 }
3925 return Register();
3926}
3927
3928Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(MVT RetVT, Register Op0) {
3929 if (RetVT.SimpleTy != MVT::v8bf16)
3930 return Register();
3931 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3932 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
3933 }
3934 return Register();
3935}
3936
3937Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(MVT RetVT, Register Op0) {
3938 if (RetVT.SimpleTy != MVT::v16bf16)
3939 return Register();
3940 if ((Subtarget->hasBF16())) {
3941 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
3942 }
3943 return Register();
3944}
3945
3946Register fastEmit_X86ISD_CVTNEPS2BF16_r(MVT VT, MVT RetVT, Register Op0) {
3947 switch (VT.SimpleTy) {
3948 case MVT::v4f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(RetVT, Op0);
3949 case MVT::v8f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(RetVT, Op0);
3950 case MVT::v16f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(RetVT, Op0);
3951 default: return Register();
3952 }
3953}
3954
3955// FastEmit functions for X86ISD::CVTP2IBS.
3956
3957Register fastEmit_X86ISD_CVTP2IBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
3958 if (RetVT.SimpleTy != MVT::v8i16)
3959 return Register();
3960 if ((Subtarget->hasAVX10_2())) {
3961 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
3962 }
3963 return Register();
3964}
3965
3966Register fastEmit_X86ISD_CVTP2IBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
3967 if (RetVT.SimpleTy != MVT::v16i16)
3968 return Register();
3969 if ((Subtarget->hasAVX10_2())) {
3970 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
3971 }
3972 return Register();
3973}
3974
3975Register fastEmit_X86ISD_CVTP2IBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
3976 if (RetVT.SimpleTy != MVT::v32i16)
3977 return Register();
3978 if ((Subtarget->hasAVX10_2_512())) {
3979 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZrr, RC: &X86::VR512RegClass, Op0);
3980 }
3981 return Register();
3982}
3983
3984Register fastEmit_X86ISD_CVTP2IBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
3985 if (RetVT.SimpleTy != MVT::v8i16)
3986 return Register();
3987 if ((Subtarget->hasAVX10_2())) {
3988 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
3989 }
3990 return Register();
3991}
3992
3993Register fastEmit_X86ISD_CVTP2IBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
3994 if (RetVT.SimpleTy != MVT::v16i16)
3995 return Register();
3996 if ((Subtarget->hasAVX10_2())) {
3997 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
3998 }
3999 return Register();
4000}
4001
4002Register fastEmit_X86ISD_CVTP2IBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4003 if (RetVT.SimpleTy != MVT::v32i16)
4004 return Register();
4005 if ((Subtarget->hasAVX10_2_512())) {
4006 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZrr, RC: &X86::VR512RegClass, Op0);
4007 }
4008 return Register();
4009}
4010
4011Register fastEmit_X86ISD_CVTP2IBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4012 if (RetVT.SimpleTy != MVT::v4i32)
4013 return Register();
4014 if ((Subtarget->hasAVX10_2())) {
4015 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4016 }
4017 return Register();
4018}
4019
4020Register fastEmit_X86ISD_CVTP2IBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
4021 if (RetVT.SimpleTy != MVT::v8i32)
4022 return Register();
4023 if ((Subtarget->hasAVX10_2())) {
4024 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4025 }
4026 return Register();
4027}
4028
4029Register fastEmit_X86ISD_CVTP2IBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
4030 if (RetVT.SimpleTy != MVT::v16i32)
4031 return Register();
4032 if ((Subtarget->hasAVX10_2_512())) {
4033 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZrr, RC: &X86::VR512RegClass, Op0);
4034 }
4035 return Register();
4036}
4037
4038Register fastEmit_X86ISD_CVTP2IBS_r(MVT VT, MVT RetVT, Register Op0) {
4039 switch (VT.SimpleTy) {
4040 case MVT::v8f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v8f16_r(RetVT, Op0);
4041 case MVT::v16f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v16f16_r(RetVT, Op0);
4042 case MVT::v32f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v32f16_r(RetVT, Op0);
4043 case MVT::v8bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v8bf16_r(RetVT, Op0);
4044 case MVT::v16bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v16bf16_r(RetVT, Op0);
4045 case MVT::v32bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v32bf16_r(RetVT, Op0);
4046 case MVT::v4f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v4f32_r(RetVT, Op0);
4047 case MVT::v8f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v8f32_r(RetVT, Op0);
4048 case MVT::v16f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v16f32_r(RetVT, Op0);
4049 default: return Register();
4050 }
4051}
4052
4053// FastEmit functions for X86ISD::CVTP2IUBS.
4054
4055Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
4056 if (RetVT.SimpleTy != MVT::v8i16)
4057 return Register();
4058 if ((Subtarget->hasAVX10_2())) {
4059 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4060 }
4061 return Register();
4062}
4063
4064Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4065 if (RetVT.SimpleTy != MVT::v16i16)
4066 return Register();
4067 if ((Subtarget->hasAVX10_2())) {
4068 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4069 }
4070 return Register();
4071}
4072
4073Register fastEmit_X86ISD_CVTP2IUBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4074 if (RetVT.SimpleTy != MVT::v32i16)
4075 return Register();
4076 if ((Subtarget->hasAVX10_2_512())) {
4077 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZrr, RC: &X86::VR512RegClass, Op0);
4078 }
4079 return Register();
4080}
4081
4082Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
4083 if (RetVT.SimpleTy != MVT::v8i16)
4084 return Register();
4085 if ((Subtarget->hasAVX10_2())) {
4086 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4087 }
4088 return Register();
4089}
4090
4091Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
4092 if (RetVT.SimpleTy != MVT::v16i16)
4093 return Register();
4094 if ((Subtarget->hasAVX10_2())) {
4095 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4096 }
4097 return Register();
4098}
4099
4100Register fastEmit_X86ISD_CVTP2IUBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4101 if (RetVT.SimpleTy != MVT::v32i16)
4102 return Register();
4103 if ((Subtarget->hasAVX10_2_512())) {
4104 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZrr, RC: &X86::VR512RegClass, Op0);
4105 }
4106 return Register();
4107}
4108
4109Register fastEmit_X86ISD_CVTP2IUBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4110 if (RetVT.SimpleTy != MVT::v4i32)
4111 return Register();
4112 if ((Subtarget->hasAVX10_2())) {
4113 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4114 }
4115 return Register();
4116}
4117
4118Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
4119 if (RetVT.SimpleTy != MVT::v8i32)
4120 return Register();
4121 if ((Subtarget->hasAVX10_2())) {
4122 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4123 }
4124 return Register();
4125}
4126
4127Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
4128 if (RetVT.SimpleTy != MVT::v16i32)
4129 return Register();
4130 if ((Subtarget->hasAVX10_2_512())) {
4131 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZrr, RC: &X86::VR512RegClass, Op0);
4132 }
4133 return Register();
4134}
4135
4136Register fastEmit_X86ISD_CVTP2IUBS_r(MVT VT, MVT RetVT, Register Op0) {
4137 switch (VT.SimpleTy) {
4138 case MVT::v8f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8f16_r(RetVT, Op0);
4139 case MVT::v16f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16f16_r(RetVT, Op0);
4140 case MVT::v32f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v32f16_r(RetVT, Op0);
4141 case MVT::v8bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8bf16_r(RetVT, Op0);
4142 case MVT::v16bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16bf16_r(RetVT, Op0);
4143 case MVT::v32bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v32bf16_r(RetVT, Op0);
4144 case MVT::v4f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v4f32_r(RetVT, Op0);
4145 case MVT::v8f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8f32_r(RetVT, Op0);
4146 case MVT::v16f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16f32_r(RetVT, Op0);
4147 default: return Register();
4148 }
4149}
4150
4151// FastEmit functions for X86ISD::CVTP2SI.
4152
4153Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
4154 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4155 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
4156 }
4157 return Register();
4158}
4159
4160Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
4161 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4162 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4163 }
4164 return Register();
4165}
4166
4167Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
4168 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4169 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4170 }
4171 return Register();
4172}
4173
4174Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
4175 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4176 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4177 }
4178 return Register();
4179}
4180
4181Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
4182 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4183 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4184 }
4185 return Register();
4186}
4187
4188Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
4189 if ((Subtarget->hasFP16())) {
4190 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
4191 }
4192 return Register();
4193}
4194
4195Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4196switch (RetVT.SimpleTy) {
4197 case MVT::v8i16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
4198 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
4199 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
4200 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
4201 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
4202 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
4203 default: return Register();
4204}
4205}
4206
4207Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
4208 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
4210 }
4211 return Register();
4212}
4213
4214Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
4215 if ((Subtarget->hasFP16())) {
4216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
4217 }
4218 return Register();
4219}
4220
4221Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
4222switch (RetVT.SimpleTy) {
4223 case MVT::v16i16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
4224 case MVT::v16i32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
4225 default: return Register();
4226}
4227}
4228
4229Register fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
4230 if (RetVT.SimpleTy != MVT::v32i16)
4231 return Register();
4232 if ((Subtarget->hasFP16())) {
4233 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
4234 }
4235 return Register();
4236}
4237
4238Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
4239 if ((Subtarget->hasVLX())) {
4240 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4241 }
4242 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4243 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4244 }
4245 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4246 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4247 }
4248 return Register();
4249}
4250
4251Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
4252 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4253 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4254 }
4255 return Register();
4256}
4257
4258Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
4259 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4260 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4261 }
4262 return Register();
4263}
4264
4265Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4266switch (RetVT.SimpleTy) {
4267 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
4268 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
4269 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
4270 default: return Register();
4271}
4272}
4273
4274Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
4275 if ((Subtarget->hasVLX())) {
4276 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4277 }
4278 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4279 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
4280 }
4281 return Register();
4282}
4283
4284Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
4285 if ((Subtarget->hasDQI())) {
4286 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
4287 }
4288 return Register();
4289}
4290
4291Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
4292switch (RetVT.SimpleTy) {
4293 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
4294 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
4295 default: return Register();
4296}
4297}
4298
4299Register fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
4300 if (RetVT.SimpleTy != MVT::v16i32)
4301 return Register();
4302 if ((Subtarget->hasAVX512())) {
4303 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
4304 }
4305 return Register();
4306}
4307
4308Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
4309 if ((Subtarget->hasVLX())) {
4310 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4311 }
4312 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4313 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4314 }
4315 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4316 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4317 }
4318 return Register();
4319}
4320
4321Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
4322 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4323 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4324 }
4325 return Register();
4326}
4327
4328Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4329switch (RetVT.SimpleTy) {
4330 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
4331 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
4332 default: return Register();
4333}
4334}
4335
4336Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
4337 if ((Subtarget->hasVLX())) {
4338 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
4339 }
4340 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4341 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
4342 }
4343 return Register();
4344}
4345
4346Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
4347 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4349 }
4350 return Register();
4351}
4352
4353Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
4354switch (RetVT.SimpleTy) {
4355 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
4356 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
4357 default: return Register();
4358}
4359}
4360
4361Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
4362 if ((Subtarget->hasAVX512())) {
4363 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
4364 }
4365 return Register();
4366}
4367
4368Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
4369 if ((Subtarget->hasDQI())) {
4370 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
4371 }
4372 return Register();
4373}
4374
4375Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
4376switch (RetVT.SimpleTy) {
4377 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
4378 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
4379 default: return Register();
4380}
4381}
4382
4383Register fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
4384 switch (VT.SimpleTy) {
4385 case MVT::v8f16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(RetVT, Op0);
4386 case MVT::v16f16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(RetVT, Op0);
4387 case MVT::v32f16: return fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(RetVT, Op0);
4388 case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0);
4389 case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0);
4390 case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0);
4391 case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0);
4392 case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0);
4393 case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0);
4394 default: return Register();
4395 }
4396}
4397
4398// FastEmit functions for X86ISD::CVTP2UI.
4399
4400Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
4401 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4402 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
4403 }
4404 return Register();
4405}
4406
4407Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
4408 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4409 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4410 }
4411 return Register();
4412}
4413
4414Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
4415 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4416 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4417 }
4418 return Register();
4419}
4420
4421Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
4422 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4423 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4424 }
4425 return Register();
4426}
4427
4428Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
4429 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4430 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4431 }
4432 return Register();
4433}
4434
4435Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
4436 if ((Subtarget->hasFP16())) {
4437 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
4438 }
4439 return Register();
4440}
4441
4442Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4443switch (RetVT.SimpleTy) {
4444 case MVT::v8i16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
4445 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
4446 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
4447 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
4448 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
4449 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
4450 default: return Register();
4451}
4452}
4453
4454Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
4455 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4456 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
4457 }
4458 return Register();
4459}
4460
4461Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
4462 if ((Subtarget->hasFP16())) {
4463 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
4464 }
4465 return Register();
4466}
4467
4468Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
4469switch (RetVT.SimpleTy) {
4470 case MVT::v16i16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
4471 case MVT::v16i32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
4472 default: return Register();
4473}
4474}
4475
4476Register fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
4477 if (RetVT.SimpleTy != MVT::v32i16)
4478 return Register();
4479 if ((Subtarget->hasFP16())) {
4480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
4481 }
4482 return Register();
4483}
4484
4485Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
4486 if ((Subtarget->hasVLX())) {
4487 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4488 }
4489 return Register();
4490}
4491
4492Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
4493 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4495 }
4496 return Register();
4497}
4498
4499Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
4500 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4502 }
4503 return Register();
4504}
4505
4506Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4507switch (RetVT.SimpleTy) {
4508 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
4509 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
4510 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
4511 default: return Register();
4512}
4513}
4514
4515Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
4516 if ((Subtarget->hasVLX())) {
4517 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4518 }
4519 return Register();
4520}
4521
4522Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
4523 if ((Subtarget->hasDQI())) {
4524 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
4525 }
4526 return Register();
4527}
4528
4529Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
4530switch (RetVT.SimpleTy) {
4531 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
4532 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
4533 default: return Register();
4534}
4535}
4536
4537Register fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
4538 if (RetVT.SimpleTy != MVT::v16i32)
4539 return Register();
4540 if ((Subtarget->hasAVX512())) {
4541 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
4542 }
4543 return Register();
4544}
4545
4546Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
4547 if ((Subtarget->hasVLX())) {
4548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4549 }
4550 return Register();
4551}
4552
4553Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
4554 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4556 }
4557 return Register();
4558}
4559
4560Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4561switch (RetVT.SimpleTy) {
4562 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
4563 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
4564 default: return Register();
4565}
4566}
4567
4568Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
4569 if ((Subtarget->hasVLX())) {
4570 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
4571 }
4572 return Register();
4573}
4574
4575Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
4576 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4578 }
4579 return Register();
4580}
4581
4582Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
4583switch (RetVT.SimpleTy) {
4584 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
4585 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
4586 default: return Register();
4587}
4588}
4589
4590Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
4591 if ((Subtarget->hasAVX512())) {
4592 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
4593 }
4594 return Register();
4595}
4596
4597Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
4598 if ((Subtarget->hasDQI())) {
4599 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
4600 }
4601 return Register();
4602}
4603
4604Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
4605switch (RetVT.SimpleTy) {
4606 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
4607 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
4608 default: return Register();
4609}
4610}
4611
4612Register fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
4613 switch (VT.SimpleTy) {
4614 case MVT::v8f16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(RetVT, Op0);
4615 case MVT::v16f16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(RetVT, Op0);
4616 case MVT::v32f16: return fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(RetVT, Op0);
4617 case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0);
4618 case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0);
4619 case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0);
4620 case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0);
4621 case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0);
4622 case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0);
4623 default: return Register();
4624 }
4625}
4626
4627// FastEmit functions for X86ISD::CVTPH2PS.
4628
4629Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Register Op0) {
4630 if ((Subtarget->hasVLX())) {
4631 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4632 }
4633 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4634 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
4635 }
4636 return Register();
4637}
4638
4639Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Register Op0) {
4640 if ((Subtarget->hasVLX())) {
4641 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
4642 }
4643 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4644 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
4645 }
4646 return Register();
4647}
4648
4649Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, Register Op0) {
4650switch (RetVT.SimpleTy) {
4651 case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
4652 case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
4653 default: return Register();
4654}
4655}
4656
4657Register fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, Register Op0) {
4658 if (RetVT.SimpleTy != MVT::v16f32)
4659 return Register();
4660 if ((Subtarget->hasAVX512())) {
4661 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
4662 }
4663 return Register();
4664}
4665
4666Register fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, Register Op0) {
4667 switch (VT.SimpleTy) {
4668 case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
4669 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
4670 default: return Register();
4671 }
4672}
4673
4674// FastEmit functions for X86ISD::CVTPH2PS_SAE.
4675
4676Register fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(MVT RetVT, Register Op0) {
4677 if (RetVT.SimpleTy != MVT::v16f32)
4678 return Register();
4679 if ((Subtarget->hasAVX512())) {
4680 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrrb, RC: &X86::VR512RegClass, Op0);
4681 }
4682 return Register();
4683}
4684
4685Register fastEmit_X86ISD_CVTPH2PS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
4686 switch (VT.SimpleTy) {
4687 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(RetVT, Op0);
4688 default: return Register();
4689 }
4690}
4691
4692// FastEmit functions for X86ISD::CVTS2SI.
4693
4694Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Register Op0) {
4695 if ((Subtarget->hasFP16())) {
4696 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4697 }
4698 return Register();
4699}
4700
4701Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Register Op0) {
4702 if ((Subtarget->hasFP16())) {
4703 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4704 }
4705 return Register();
4706}
4707
4708Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4709switch (RetVT.SimpleTy) {
4710 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Op0);
4711 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Op0);
4712 default: return Register();
4713}
4714}
4715
4716Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Register Op0) {
4717 if ((Subtarget->hasAVX512())) {
4718 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4719 }
4720 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4721 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4722 }
4723 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4724 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4725 }
4726 return Register();
4727}
4728
4729Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Register Op0) {
4730 if ((Subtarget->hasAVX512())) {
4731 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4732 }
4733 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4734 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4735 }
4736 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4737 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4738 }
4739 return Register();
4740}
4741
4742Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4743switch (RetVT.SimpleTy) {
4744 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0);
4745 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0);
4746 default: return Register();
4747}
4748}
4749
4750Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Register Op0) {
4751 if ((Subtarget->hasAVX512())) {
4752 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4753 }
4754 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4755 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4756 }
4757 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4758 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4759 }
4760 return Register();
4761}
4762
4763Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Register Op0) {
4764 if ((Subtarget->hasAVX512())) {
4765 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4766 }
4767 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4768 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4769 }
4770 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4771 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4772 }
4773 return Register();
4774}
4775
4776Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4777switch (RetVT.SimpleTy) {
4778 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0);
4779 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0);
4780 default: return Register();
4781}
4782}
4783
4784Register fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, Register Op0) {
4785 switch (VT.SimpleTy) {
4786 case MVT::v8f16: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(RetVT, Op0);
4787 case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0);
4788 case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0);
4789 default: return Register();
4790 }
4791}
4792
4793// FastEmit functions for X86ISD::CVTS2UI.
4794
4795Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Register Op0) {
4796 if ((Subtarget->hasFP16())) {
4797 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4798 }
4799 return Register();
4800}
4801
4802Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Register Op0) {
4803 if ((Subtarget->hasFP16())) {
4804 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4805 }
4806 return Register();
4807}
4808
4809Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4810switch (RetVT.SimpleTy) {
4811 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Op0);
4812 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Op0);
4813 default: return Register();
4814}
4815}
4816
4817Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Register Op0) {
4818 if ((Subtarget->hasAVX512())) {
4819 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4820 }
4821 return Register();
4822}
4823
4824Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Register Op0) {
4825 if ((Subtarget->hasAVX512())) {
4826 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4827 }
4828 return Register();
4829}
4830
4831Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4832switch (RetVT.SimpleTy) {
4833 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0);
4834 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0);
4835 default: return Register();
4836}
4837}
4838
4839Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Register Op0) {
4840 if ((Subtarget->hasAVX512())) {
4841 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4842 }
4843 return Register();
4844}
4845
4846Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Register Op0) {
4847 if ((Subtarget->hasAVX512())) {
4848 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4849 }
4850 return Register();
4851}
4852
4853Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4854switch (RetVT.SimpleTy) {
4855 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0);
4856 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0);
4857 default: return Register();
4858}
4859}
4860
4861Register fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, Register Op0) {
4862 switch (VT.SimpleTy) {
4863 case MVT::v8f16: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(RetVT, Op0);
4864 case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0);
4865 case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0);
4866 default: return Register();
4867 }
4868}
4869
4870// FastEmit functions for X86ISD::CVTSI2P.
4871
4872Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
4873 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4874 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4875 }
4876 return Register();
4877}
4878
4879Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
4880 if ((Subtarget->hasVLX())) {
4881 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
4882 }
4883 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4884 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4885 }
4886 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4887 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4888 }
4889 return Register();
4890}
4891
4892Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
4893switch (RetVT.SimpleTy) {
4894 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
4895 case MVT::v2f64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
4896 default: return Register();
4897}
4898}
4899
4900Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
4901 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4902 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4903 }
4904 return Register();
4905}
4906
4907Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
4908 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4909 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4910 }
4911 return Register();
4912}
4913
4914Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
4915switch (RetVT.SimpleTy) {
4916 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
4917 case MVT::v4f32: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
4918 default: return Register();
4919}
4920}
4921
4922Register fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
4923 if (RetVT.SimpleTy != MVT::v8f16)
4924 return Register();
4925 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4926 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
4927 }
4928 return Register();
4929}
4930
4931Register fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, Register Op0) {
4932 switch (VT.SimpleTy) {
4933 case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
4934 case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
4935 case MVT::v4i64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
4936 default: return Register();
4937 }
4938}
4939
4940// FastEmit functions for X86ISD::CVTTP2IBS.
4941
4942Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
4943 if (RetVT.SimpleTy != MVT::v8i16)
4944 return Register();
4945 if ((Subtarget->hasAVX10_2())) {
4946 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4947 }
4948 return Register();
4949}
4950
4951Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4952 if (RetVT.SimpleTy != MVT::v16i16)
4953 return Register();
4954 if ((Subtarget->hasAVX10_2())) {
4955 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4956 }
4957 return Register();
4958}
4959
4960Register fastEmit_X86ISD_CVTTP2IBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4961 if (RetVT.SimpleTy != MVT::v32i16)
4962 return Register();
4963 if ((Subtarget->hasAVX10_2_512())) {
4964 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZrr, RC: &X86::VR512RegClass, Op0);
4965 }
4966 return Register();
4967}
4968
4969Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
4970 if (RetVT.SimpleTy != MVT::v8i16)
4971 return Register();
4972 if ((Subtarget->hasAVX10_2())) {
4973 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4974 }
4975 return Register();
4976}
4977
4978Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
4979 if (RetVT.SimpleTy != MVT::v16i16)
4980 return Register();
4981 if ((Subtarget->hasAVX10_2())) {
4982 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4983 }
4984 return Register();
4985}
4986
4987Register fastEmit_X86ISD_CVTTP2IBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4988 if (RetVT.SimpleTy != MVT::v32i16)
4989 return Register();
4990 if ((Subtarget->hasAVX10_2_512())) {
4991 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZrr, RC: &X86::VR512RegClass, Op0);
4992 }
4993 return Register();
4994}
4995
4996Register fastEmit_X86ISD_CVTTP2IBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4997 if (RetVT.SimpleTy != MVT::v4i32)
4998 return Register();
4999 if ((Subtarget->hasAVX10_2())) {
5000 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5001 }
5002 return Register();
5003}
5004
5005Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5006 if (RetVT.SimpleTy != MVT::v8i32)
5007 return Register();
5008 if ((Subtarget->hasAVX10_2())) {
5009 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5010 }
5011 return Register();
5012}
5013
5014Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5015 if (RetVT.SimpleTy != MVT::v16i32)
5016 return Register();
5017 if ((Subtarget->hasAVX10_2_512())) {
5018 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZrr, RC: &X86::VR512RegClass, Op0);
5019 }
5020 return Register();
5021}
5022
5023Register fastEmit_X86ISD_CVTTP2IBS_r(MVT VT, MVT RetVT, Register Op0) {
5024 switch (VT.SimpleTy) {
5025 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8f16_r(RetVT, Op0);
5026 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16f16_r(RetVT, Op0);
5027 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v32f16_r(RetVT, Op0);
5028 case MVT::v8bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8bf16_r(RetVT, Op0);
5029 case MVT::v16bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16bf16_r(RetVT, Op0);
5030 case MVT::v32bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v32bf16_r(RetVT, Op0);
5031 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v4f32_r(RetVT, Op0);
5032 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8f32_r(RetVT, Op0);
5033 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16f32_r(RetVT, Op0);
5034 default: return Register();
5035 }
5036}
5037
5038// FastEmit functions for X86ISD::CVTTP2IBS_SAE.
5039
5040Register fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5041 if (RetVT.SimpleTy != MVT::v32i16)
5042 return Register();
5043 if ((Subtarget->hasAVX10_2_512())) {
5044 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZrrb, RC: &X86::VR512RegClass, Op0);
5045 }
5046 return Register();
5047}
5048
5049Register fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5050 if (RetVT.SimpleTy != MVT::v16i32)
5051 return Register();
5052 if ((Subtarget->hasAVX10_2_512())) {
5053 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZrrb, RC: &X86::VR512RegClass, Op0);
5054 }
5055 return Register();
5056}
5057
5058Register fastEmit_X86ISD_CVTTP2IBS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5059 switch (VT.SimpleTy) {
5060 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v32f16_r(RetVT, Op0);
5061 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v16f32_r(RetVT, Op0);
5062 default: return Register();
5063 }
5064}
5065
5066// FastEmit functions for X86ISD::CVTTP2IUBS.
5067
5068Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
5069 if (RetVT.SimpleTy != MVT::v8i16)
5070 return Register();
5071 if ((Subtarget->hasAVX10_2())) {
5072 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5073 }
5074 return Register();
5075}
5076
5077Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
5078 if (RetVT.SimpleTy != MVT::v16i16)
5079 return Register();
5080 if ((Subtarget->hasAVX10_2())) {
5081 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5082 }
5083 return Register();
5084}
5085
5086Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
5087 if (RetVT.SimpleTy != MVT::v32i16)
5088 return Register();
5089 if ((Subtarget->hasAVX10_2_512())) {
5090 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZrr, RC: &X86::VR512RegClass, Op0);
5091 }
5092 return Register();
5093}
5094
5095Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
5096 if (RetVT.SimpleTy != MVT::v8i16)
5097 return Register();
5098 if ((Subtarget->hasAVX10_2())) {
5099 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5100 }
5101 return Register();
5102}
5103
5104Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
5105 if (RetVT.SimpleTy != MVT::v16i16)
5106 return Register();
5107 if ((Subtarget->hasAVX10_2())) {
5108 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5109 }
5110 return Register();
5111}
5112
5113Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
5114 if (RetVT.SimpleTy != MVT::v32i16)
5115 return Register();
5116 if ((Subtarget->hasAVX10_2_512())) {
5117 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZrr, RC: &X86::VR512RegClass, Op0);
5118 }
5119 return Register();
5120}
5121
5122Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5123 if (RetVT.SimpleTy != MVT::v4i32)
5124 return Register();
5125 if ((Subtarget->hasAVX10_2())) {
5126 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5127 }
5128 return Register();
5129}
5130
5131Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5132 if (RetVT.SimpleTy != MVT::v8i32)
5133 return Register();
5134 if ((Subtarget->hasAVX10_2())) {
5135 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5136 }
5137 return Register();
5138}
5139
5140Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5141 if (RetVT.SimpleTy != MVT::v16i32)
5142 return Register();
5143 if ((Subtarget->hasAVX10_2_512())) {
5144 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZrr, RC: &X86::VR512RegClass, Op0);
5145 }
5146 return Register();
5147}
5148
5149Register fastEmit_X86ISD_CVTTP2IUBS_r(MVT VT, MVT RetVT, Register Op0) {
5150 switch (VT.SimpleTy) {
5151 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f16_r(RetVT, Op0);
5152 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f16_r(RetVT, Op0);
5153 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v32f16_r(RetVT, Op0);
5154 case MVT::v8bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8bf16_r(RetVT, Op0);
5155 case MVT::v16bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16bf16_r(RetVT, Op0);
5156 case MVT::v32bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v32bf16_r(RetVT, Op0);
5157 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v4f32_r(RetVT, Op0);
5158 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f32_r(RetVT, Op0);
5159 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f32_r(RetVT, Op0);
5160 default: return Register();
5161 }
5162}
5163
5164// FastEmit functions for X86ISD::CVTTP2IUBS_SAE.
5165
5166Register fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5167 if (RetVT.SimpleTy != MVT::v32i16)
5168 return Register();
5169 if ((Subtarget->hasAVX10_2_512())) {
5170 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZrrb, RC: &X86::VR512RegClass, Op0);
5171 }
5172 return Register();
5173}
5174
5175Register fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5176 if (RetVT.SimpleTy != MVT::v16i32)
5177 return Register();
5178 if ((Subtarget->hasAVX10_2_512())) {
5179 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZrrb, RC: &X86::VR512RegClass, Op0);
5180 }
5181 return Register();
5182}
5183
5184Register fastEmit_X86ISD_CVTTP2IUBS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5185 switch (VT.SimpleTy) {
5186 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v32f16_r(RetVT, Op0);
5187 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v16f32_r(RetVT, Op0);
5188 default: return Register();
5189 }
5190}
5191
5192// FastEmit functions for X86ISD::CVTTP2SI.
5193
5194Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
5195 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5196 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
5197 }
5198 return Register();
5199}
5200
5201Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
5202 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5203 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5204 }
5205 return Register();
5206}
5207
5208Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
5209 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5210 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
5211 }
5212 return Register();
5213}
5214
5215Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
5216 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5217 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5218 }
5219 return Register();
5220}
5221
5222Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
5223 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5224 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5225 }
5226 return Register();
5227}
5228
5229Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
5230 if ((Subtarget->hasFP16())) {
5231 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
5232 }
5233 return Register();
5234}
5235
5236Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
5237switch (RetVT.SimpleTy) {
5238 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
5239 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
5240 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
5241 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
5242 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
5243 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
5244 default: return Register();
5245}
5246}
5247
5248Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
5249 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5250 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
5251 }
5252 return Register();
5253}
5254
5255Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
5256 if ((Subtarget->hasFP16())) {
5257 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
5258 }
5259 return Register();
5260}
5261
5262Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
5263switch (RetVT.SimpleTy) {
5264 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
5265 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
5266 default: return Register();
5267}
5268}
5269
5270Register fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
5271 if (RetVT.SimpleTy != MVT::v32i16)
5272 return Register();
5273 if ((Subtarget->hasFP16())) {
5274 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
5275 }
5276 return Register();
5277}
5278
5279Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5280 if ((Subtarget->hasVLX())) {
5281 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5282 }
5283 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5284 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
5285 }
5286 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5287 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
5288 }
5289 return Register();
5290}
5291
5292Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5293 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5294 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5295 }
5296 return Register();
5297}
5298
5299Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5300 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5301 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5302 }
5303 return Register();
5304}
5305
5306Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
5307switch (RetVT.SimpleTy) {
5308 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
5309 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
5310 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
5311 default: return Register();
5312}
5313}
5314
5315Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5316 if ((Subtarget->hasVLX())) {
5317 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
5318 }
5319 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5320 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
5321 }
5322 return Register();
5323}
5324
5325Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5326 if ((Subtarget->hasDQI())) {
5327 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
5328 }
5329 return Register();
5330}
5331
5332Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
5333switch (RetVT.SimpleTy) {
5334 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
5335 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
5336 default: return Register();
5337}
5338}
5339
5340Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
5341 if (RetVT.SimpleTy != MVT::v16i32)
5342 return Register();
5343 if ((Subtarget->hasAVX512())) {
5344 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
5345 }
5346 return Register();
5347}
5348
5349Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5350 if ((Subtarget->hasVLX())) {
5351 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5352 }
5353 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5354 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
5355 }
5356 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
5358 }
5359 return Register();
5360}
5361
5362Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5363 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5364 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5365 }
5366 return Register();
5367}
5368
5369Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
5370switch (RetVT.SimpleTy) {
5371 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
5372 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
5373 default: return Register();
5374}
5375}
5376
5377Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5378 if ((Subtarget->hasVLX())) {
5379 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
5380 }
5381 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5382 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
5383 }
5384 return Register();
5385}
5386
5387Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5388 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5389 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5390 }
5391 return Register();
5392}
5393
5394Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
5395switch (RetVT.SimpleTy) {
5396 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
5397 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
5398 default: return Register();
5399}
5400}
5401
5402Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5403 if ((Subtarget->hasAVX512())) {
5404 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
5405 }
5406 return Register();
5407}
5408
5409Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5410 if ((Subtarget->hasDQI())) {
5411 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
5412 }
5413 return Register();
5414}
5415
5416Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
5417switch (RetVT.SimpleTy) {
5418 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
5419 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
5420 default: return Register();
5421}
5422}
5423
5424Register fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
5425 switch (VT.SimpleTy) {
5426 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
5427 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
5428 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
5429 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
5430 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
5431 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
5432 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
5433 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
5434 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
5435 default: return Register();
5436 }
5437}
5438
5439// FastEmit functions for X86ISD::CVTTP2SIS.
5440
5441Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5442 if ((Subtarget->hasAVX10_2())) {
5443 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5444 }
5445 return Register();
5446}
5447
5448Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5449 if ((Subtarget->hasAVX10_2())) {
5450 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5451 }
5452 return Register();
5453}
5454
5455Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5456 if ((Subtarget->hasAVX10_2())) {
5457 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5458 }
5459 return Register();
5460}
5461
5462Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5463switch (RetVT.SimpleTy) {
5464 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i32_r(Op0);
5465 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v2i64_r(Op0);
5466 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i64_r(Op0);
5467 default: return Register();
5468}
5469}
5470
5471Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5472 if ((Subtarget->hasAVX10_2())) {
5473 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5474 }
5475 return Register();
5476}
5477
5478Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5479 if ((Subtarget->hasAVX10_2_512())) {
5480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZrr, RC: &X86::VR512RegClass, Op0);
5481 }
5482 return Register();
5483}
5484
5485Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5486switch (RetVT.SimpleTy) {
5487 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i32_r(Op0);
5488 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i64_r(Op0);
5489 default: return Register();
5490}
5491}
5492
5493Register fastEmit_X86ISD_CVTTP2SIS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5494 if (RetVT.SimpleTy != MVT::v16i32)
5495 return Register();
5496 if ((Subtarget->hasAVX10_2_512())) {
5497 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZrr, RC: &X86::VR512RegClass, Op0);
5498 }
5499 return Register();
5500}
5501
5502Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5503 if ((Subtarget->hasAVX10_2())) {
5504 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5505 }
5506 return Register();
5507}
5508
5509Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5510 if ((Subtarget->hasAVX10_2())) {
5511 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5512 }
5513 return Register();
5514}
5515
5516Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
5517switch (RetVT.SimpleTy) {
5518 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v4i32_r(Op0);
5519 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v2i64_r(Op0);
5520 default: return Register();
5521}
5522}
5523
5524Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5525 if ((Subtarget->hasAVX10_2())) {
5526 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ256rr, RC: &X86::VR128XRegClass, Op0);
5527 }
5528 return Register();
5529}
5530
5531Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5532 if ((Subtarget->hasAVX10_2())) {
5533 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5534 }
5535 return Register();
5536}
5537
5538Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_r(MVT RetVT, Register Op0) {
5539switch (RetVT.SimpleTy) {
5540 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i32_r(Op0);
5541 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i64_r(Op0);
5542 default: return Register();
5543}
5544}
5545
5546Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5547 if ((Subtarget->hasAVX10_2_512())) {
5548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZrr, RC: &X86::VR256XRegClass, Op0);
5549 }
5550 return Register();
5551}
5552
5553Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5554 if ((Subtarget->hasAVX10_2_512())) {
5555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZrr, RC: &X86::VR512RegClass, Op0);
5556 }
5557 return Register();
5558}
5559
5560Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_r(MVT RetVT, Register Op0) {
5561switch (RetVT.SimpleTy) {
5562 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i32_r(Op0);
5563 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i64_r(Op0);
5564 default: return Register();
5565}
5566}
5567
5568Register fastEmit_X86ISD_CVTTP2SIS_r(MVT VT, MVT RetVT, Register Op0) {
5569 switch (VT.SimpleTy) {
5570 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_r(RetVT, Op0);
5571 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_r(RetVT, Op0);
5572 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v16f32_r(RetVT, Op0);
5573 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_r(RetVT, Op0);
5574 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_r(RetVT, Op0);
5575 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_r(RetVT, Op0);
5576 default: return Register();
5577 }
5578}
5579
5580// FastEmit functions for X86ISD::CVTTP2SIS_SAE.
5581
5582Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
5583 if (RetVT.SimpleTy != MVT::v4i64)
5584 return Register();
5585 if ((Subtarget->hasAVX10_2())) {
5586 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
5587 }
5588 return Register();
5589}
5590
5591Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
5592 if (RetVT.SimpleTy != MVT::v8i64)
5593 return Register();
5594 if ((Subtarget->hasAVX10_2_512())) {
5595 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZrrb, RC: &X86::VR512RegClass, Op0);
5596 }
5597 return Register();
5598}
5599
5600Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5601 if (RetVT.SimpleTy != MVT::v16i32)
5602 return Register();
5603 if ((Subtarget->hasAVX10_2_512())) {
5604 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZrrb, RC: &X86::VR512RegClass, Op0);
5605 }
5606 return Register();
5607}
5608
5609Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5610 if ((Subtarget->hasAVX10_2())) {
5611 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ256rrb, RC: &X86::VR128XRegClass, Op0);
5612 }
5613 return Register();
5614}
5615
5616Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5617 if ((Subtarget->hasAVX10_2())) {
5618 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
5619 }
5620 return Register();
5621}
5622
5623Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_r(MVT RetVT, Register Op0) {
5624switch (RetVT.SimpleTy) {
5625 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i32_r(Op0);
5626 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i64_r(Op0);
5627 default: return Register();
5628}
5629}
5630
5631Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5632 if ((Subtarget->hasAVX10_2_512())) {
5633 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZrrb, RC: &X86::VR256XRegClass, Op0);
5634 }
5635 return Register();
5636}
5637
5638Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5639 if ((Subtarget->hasAVX10_2_512())) {
5640 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZrrb, RC: &X86::VR512RegClass, Op0);
5641 }
5642 return Register();
5643}
5644
5645Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
5646switch (RetVT.SimpleTy) {
5647 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5648 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5649 default: return Register();
5650}
5651}
5652
5653Register fastEmit_X86ISD_CVTTP2SIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5654 switch (VT.SimpleTy) {
5655 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f32_r(RetVT, Op0);
5656 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f32_r(RetVT, Op0);
5657 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v16f32_r(RetVT, Op0);
5658 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_r(RetVT, Op0);
5659 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_r(RetVT, Op0);
5660 default: return Register();
5661 }
5662}
5663
5664// FastEmit functions for X86ISD::CVTTP2SI_SAE.
5665
5666Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
5667 if (RetVT.SimpleTy != MVT::v8i64)
5668 return Register();
5669 if ((Subtarget->hasFP16())) {
5670 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrrb, RC: &X86::VR512RegClass, Op0);
5671 }
5672 return Register();
5673}
5674
5675Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
5676 if (RetVT.SimpleTy != MVT::v16i32)
5677 return Register();
5678 if ((Subtarget->hasFP16())) {
5679 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrrb, RC: &X86::VR512RegClass, Op0);
5680 }
5681 return Register();
5682}
5683
5684Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5685 if (RetVT.SimpleTy != MVT::v32i16)
5686 return Register();
5687 if ((Subtarget->hasFP16())) {
5688 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrrb, RC: &X86::VR512RegClass, Op0);
5689 }
5690 return Register();
5691}
5692
5693Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
5694 if (RetVT.SimpleTy != MVT::v8i64)
5695 return Register();
5696 if ((Subtarget->hasDQI())) {
5697 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrrb, RC: &X86::VR512RegClass, Op0);
5698 }
5699 return Register();
5700}
5701
5702Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5703 if (RetVT.SimpleTy != MVT::v16i32)
5704 return Register();
5705 if ((Subtarget->hasAVX512())) {
5706 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrrb, RC: &X86::VR512RegClass, Op0);
5707 }
5708 return Register();
5709}
5710
5711Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5712 if ((Subtarget->hasAVX512())) {
5713 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrrb, RC: &X86::VR256XRegClass, Op0);
5714 }
5715 return Register();
5716}
5717
5718Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5719 if ((Subtarget->hasDQI())) {
5720 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrrb, RC: &X86::VR512RegClass, Op0);
5721 }
5722 return Register();
5723}
5724
5725Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
5726switch (RetVT.SimpleTy) {
5727 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5728 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5729 default: return Register();
5730}
5731}
5732
5733Register fastEmit_X86ISD_CVTTP2SI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5734 switch (VT.SimpleTy) {
5735 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(RetVT, Op0);
5736 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(RetVT, Op0);
5737 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(RetVT, Op0);
5738 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(RetVT, Op0);
5739 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(RetVT, Op0);
5740 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(RetVT, Op0);
5741 default: return Register();
5742 }
5743}
5744
5745// FastEmit functions for X86ISD::CVTTP2UI.
5746
5747Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
5748 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5749 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
5750 }
5751 return Register();
5752}
5753
5754Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
5755 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5756 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5757 }
5758 return Register();
5759}
5760
5761Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
5762 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5763 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5764 }
5765 return Register();
5766}
5767
5768Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
5769 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5770 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5771 }
5772 return Register();
5773}
5774
5775Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
5776 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5777 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5778 }
5779 return Register();
5780}
5781
5782Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
5783 if ((Subtarget->hasFP16())) {
5784 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
5785 }
5786 return Register();
5787}
5788
5789Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
5790switch (RetVT.SimpleTy) {
5791 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
5792 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
5793 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
5794 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
5795 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
5796 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
5797 default: return Register();
5798}
5799}
5800
5801Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
5802 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5803 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
5804 }
5805 return Register();
5806}
5807
5808Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
5809 if ((Subtarget->hasFP16())) {
5810 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
5811 }
5812 return Register();
5813}
5814
5815Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
5816switch (RetVT.SimpleTy) {
5817 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
5818 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
5819 default: return Register();
5820}
5821}
5822
5823Register fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
5824 if (RetVT.SimpleTy != MVT::v32i16)
5825 return Register();
5826 if ((Subtarget->hasFP16())) {
5827 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
5828 }
5829 return Register();
5830}
5831
5832Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5833 if ((Subtarget->hasVLX())) {
5834 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5835 }
5836 return Register();
5837}
5838
5839Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5840 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5841 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5842 }
5843 return Register();
5844}
5845
5846Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5847 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5848 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5849 }
5850 return Register();
5851}
5852
5853Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
5854switch (RetVT.SimpleTy) {
5855 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
5856 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
5857 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
5858 default: return Register();
5859}
5860}
5861
5862Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5863 if ((Subtarget->hasVLX())) {
5864 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5865 }
5866 return Register();
5867}
5868
5869Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5870 if ((Subtarget->hasDQI())) {
5871 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
5872 }
5873 return Register();
5874}
5875
5876Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
5877switch (RetVT.SimpleTy) {
5878 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
5879 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
5880 default: return Register();
5881}
5882}
5883
5884Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
5885 if (RetVT.SimpleTy != MVT::v16i32)
5886 return Register();
5887 if ((Subtarget->hasAVX512())) {
5888 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
5889 }
5890 return Register();
5891}
5892
5893Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5894 if ((Subtarget->hasVLX())) {
5895 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5896 }
5897 return Register();
5898}
5899
5900Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5901 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5902 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5903 }
5904 return Register();
5905}
5906
5907Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
5908switch (RetVT.SimpleTy) {
5909 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
5910 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
5911 default: return Register();
5912}
5913}
5914
5915Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5916 if ((Subtarget->hasVLX())) {
5917 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
5918 }
5919 return Register();
5920}
5921
5922Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5923 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5924 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5925 }
5926 return Register();
5927}
5928
5929Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
5930switch (RetVT.SimpleTy) {
5931 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
5932 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
5933 default: return Register();
5934}
5935}
5936
5937Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5938 if ((Subtarget->hasAVX512())) {
5939 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
5940 }
5941 return Register();
5942}
5943
5944Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5945 if ((Subtarget->hasDQI())) {
5946 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
5947 }
5948 return Register();
5949}
5950
5951Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
5952switch (RetVT.SimpleTy) {
5953 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
5954 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
5955 default: return Register();
5956}
5957}
5958
5959Register fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
5960 switch (VT.SimpleTy) {
5961 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
5962 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
5963 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
5964 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
5965 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
5966 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
5967 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
5968 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
5969 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
5970 default: return Register();
5971 }
5972}
5973
5974// FastEmit functions for X86ISD::CVTTP2UIS.
5975
5976Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5977 if ((Subtarget->hasAVX10_2())) {
5978 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5979 }
5980 return Register();
5981}
5982
5983Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5984 if ((Subtarget->hasAVX10_2())) {
5985 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5986 }
5987 return Register();
5988}
5989
5990Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5991 if ((Subtarget->hasAVX10_2())) {
5992 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5993 }
5994 return Register();
5995}
5996
5997Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5998switch (RetVT.SimpleTy) {
5999 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i32_r(Op0);
6000 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v2i64_r(Op0);
6001 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i64_r(Op0);
6002 default: return Register();
6003}
6004}
6005
6006Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i32_r(Register Op0) {
6007 if ((Subtarget->hasAVX10_2())) {
6008 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZ256rr, RC: &X86::VR256XRegClass, Op0);
6009 }
6010 return Register();
6011}
6012
6013Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i64_r(Register Op0) {
6014 if ((Subtarget->hasAVX10_2_512())) {
6015 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZrr, RC: &X86::VR512RegClass, Op0);
6016 }
6017 return Register();
6018}
6019
6020Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_r(MVT RetVT, Register Op0) {
6021switch (RetVT.SimpleTy) {
6022 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i32_r(Op0);
6023 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i64_r(Op0);
6024 default: return Register();
6025}
6026}
6027
6028Register fastEmit_X86ISD_CVTTP2UIS_MVT_v16f32_r(MVT RetVT, Register Op0) {
6029 if (RetVT.SimpleTy != MVT::v16i32)
6030 return Register();
6031 if ((Subtarget->hasAVX10_2_512())) {
6032 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZrr, RC: &X86::VR512RegClass, Op0);
6033 }
6034 return Register();
6035}
6036
6037Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v4i32_r(Register Op0) {
6038 if ((Subtarget->hasAVX10_2())) {
6039 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6040 }
6041 return Register();
6042}
6043
6044Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v2i64_r(Register Op0) {
6045 if ((Subtarget->hasAVX10_2())) {
6046 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6047 }
6048 return Register();
6049}
6050
6051Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6052switch (RetVT.SimpleTy) {
6053 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v4i32_r(Op0);
6054 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v2i64_r(Op0);
6055 default: return Register();
6056}
6057}
6058
6059Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i32_r(Register Op0) {
6060 if ((Subtarget->hasAVX10_2())) {
6061 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ256rr, RC: &X86::VR128XRegClass, Op0);
6062 }
6063 return Register();
6064}
6065
6066Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i64_r(Register Op0) {
6067 if ((Subtarget->hasAVX10_2())) {
6068 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ256rr, RC: &X86::VR256XRegClass, Op0);
6069 }
6070 return Register();
6071}
6072
6073Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_r(MVT RetVT, Register Op0) {
6074switch (RetVT.SimpleTy) {
6075 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i32_r(Op0);
6076 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i64_r(Op0);
6077 default: return Register();
6078}
6079}
6080
6081Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6082 if ((Subtarget->hasAVX10_2_512())) {
6083 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZrr, RC: &X86::VR256XRegClass, Op0);
6084 }
6085 return Register();
6086}
6087
6088Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6089 if ((Subtarget->hasAVX10_2_512())) {
6090 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZrr, RC: &X86::VR512RegClass, Op0);
6091 }
6092 return Register();
6093}
6094
6095Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_r(MVT RetVT, Register Op0) {
6096switch (RetVT.SimpleTy) {
6097 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i32_r(Op0);
6098 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i64_r(Op0);
6099 default: return Register();
6100}
6101}
6102
6103Register fastEmit_X86ISD_CVTTP2UIS_r(MVT VT, MVT RetVT, Register Op0) {
6104 switch (VT.SimpleTy) {
6105 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_r(RetVT, Op0);
6106 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_r(RetVT, Op0);
6107 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v16f32_r(RetVT, Op0);
6108 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_r(RetVT, Op0);
6109 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_r(RetVT, Op0);
6110 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_r(RetVT, Op0);
6111 default: return Register();
6112 }
6113}
6114
6115// FastEmit functions for X86ISD::CVTTP2UIS_SAE.
6116
6117Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6118 if (RetVT.SimpleTy != MVT::v4i64)
6119 return Register();
6120 if ((Subtarget->hasAVX10_2())) {
6121 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
6122 }
6123 return Register();
6124}
6125
6126Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
6127 if (RetVT.SimpleTy != MVT::v8i64)
6128 return Register();
6129 if ((Subtarget->hasAVX10_2_512())) {
6130 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZrrb, RC: &X86::VR512RegClass, Op0);
6131 }
6132 return Register();
6133}
6134
6135Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6136 if (RetVT.SimpleTy != MVT::v16i32)
6137 return Register();
6138 if ((Subtarget->hasAVX10_2_512())) {
6139 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZrrb, RC: &X86::VR512RegClass, Op0);
6140 }
6141 return Register();
6142}
6143
6144Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i32_r(Register Op0) {
6145 if ((Subtarget->hasAVX10_2())) {
6146 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ256rrb, RC: &X86::VR128XRegClass, Op0);
6147 }
6148 return Register();
6149}
6150
6151Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i64_r(Register Op0) {
6152 if ((Subtarget->hasAVX10_2())) {
6153 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
6154 }
6155 return Register();
6156}
6157
6158Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_r(MVT RetVT, Register Op0) {
6159switch (RetVT.SimpleTy) {
6160 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i32_r(Op0);
6161 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i64_r(Op0);
6162 default: return Register();
6163}
6164}
6165
6166Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6167 if ((Subtarget->hasAVX10_2_512())) {
6168 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZrrb, RC: &X86::VR256XRegClass, Op0);
6169 }
6170 return Register();
6171}
6172
6173Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6174 if ((Subtarget->hasAVX10_2_512())) {
6175 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZrrb, RC: &X86::VR512RegClass, Op0);
6176 }
6177 return Register();
6178}
6179
6180Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6181switch (RetVT.SimpleTy) {
6182 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
6183 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
6184 default: return Register();
6185}
6186}
6187
6188Register fastEmit_X86ISD_CVTTP2UIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6189 switch (VT.SimpleTy) {
6190 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f32_r(RetVT, Op0);
6191 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f32_r(RetVT, Op0);
6192 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v16f32_r(RetVT, Op0);
6193 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_r(RetVT, Op0);
6194 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_r(RetVT, Op0);
6195 default: return Register();
6196 }
6197}
6198
6199// FastEmit functions for X86ISD::CVTTP2UI_SAE.
6200
6201Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6202 if (RetVT.SimpleTy != MVT::v8i64)
6203 return Register();
6204 if ((Subtarget->hasFP16())) {
6205 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6206 }
6207 return Register();
6208}
6209
6210Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
6211 if (RetVT.SimpleTy != MVT::v16i32)
6212 return Register();
6213 if ((Subtarget->hasFP16())) {
6214 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrrb, RC: &X86::VR512RegClass, Op0);
6215 }
6216 return Register();
6217}
6218
6219Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
6220 if (RetVT.SimpleTy != MVT::v32i16)
6221 return Register();
6222 if ((Subtarget->hasFP16())) {
6223 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrrb, RC: &X86::VR512RegClass, Op0);
6224 }
6225 return Register();
6226}
6227
6228Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
6229 if (RetVT.SimpleTy != MVT::v8i64)
6230 return Register();
6231 if ((Subtarget->hasDQI())) {
6232 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6233 }
6234 return Register();
6235}
6236
6237Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6238 if (RetVT.SimpleTy != MVT::v16i32)
6239 return Register();
6240 if ((Subtarget->hasAVX512())) {
6241 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrrb, RC: &X86::VR512RegClass, Op0);
6242 }
6243 return Register();
6244}
6245
6246Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6247 if ((Subtarget->hasAVX512())) {
6248 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrrb, RC: &X86::VR256XRegClass, Op0);
6249 }
6250 return Register();
6251}
6252
6253Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6254 if ((Subtarget->hasDQI())) {
6255 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6256 }
6257 return Register();
6258}
6259
6260Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6261switch (RetVT.SimpleTy) {
6262 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
6263 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
6264 default: return Register();
6265}
6266}
6267
6268Register fastEmit_X86ISD_CVTTP2UI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6269 switch (VT.SimpleTy) {
6270 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(RetVT, Op0);
6271 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(RetVT, Op0);
6272 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(RetVT, Op0);
6273 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(RetVT, Op0);
6274 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(RetVT, Op0);
6275 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(RetVT, Op0);
6276 default: return Register();
6277 }
6278}
6279
6280// FastEmit functions for X86ISD::CVTTS2SI.
6281
6282Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Register Op0) {
6283 if ((Subtarget->hasFP16())) {
6284 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6285 }
6286 return Register();
6287}
6288
6289Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Register Op0) {
6290 if ((Subtarget->hasFP16())) {
6291 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6292 }
6293 return Register();
6294}
6295
6296Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
6297switch (RetVT.SimpleTy) {
6298 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Op0);
6299 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Op0);
6300 default: return Register();
6301}
6302}
6303
6304Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Register Op0) {
6305 if ((Subtarget->hasAVX512())) {
6306 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6307 }
6308 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6309 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6310 }
6311 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6312 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6313 }
6314 return Register();
6315}
6316
6317Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Register Op0) {
6318 if ((Subtarget->hasAVX512())) {
6319 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6320 }
6321 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6322 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6323 }
6324 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6325 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6326 }
6327 return Register();
6328}
6329
6330Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
6331switch (RetVT.SimpleTy) {
6332 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0);
6333 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0);
6334 default: return Register();
6335}
6336}
6337
6338Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Register Op0) {
6339 if ((Subtarget->hasAVX512())) {
6340 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6341 }
6342 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6343 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6344 }
6345 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6346 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6347 }
6348 return Register();
6349}
6350
6351Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Register Op0) {
6352 if ((Subtarget->hasAVX512())) {
6353 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6354 }
6355 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6356 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6357 }
6358 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6359 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6360 }
6361 return Register();
6362}
6363
6364Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
6365switch (RetVT.SimpleTy) {
6366 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0);
6367 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0);
6368 default: return Register();
6369}
6370}
6371
6372Register fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, Register Op0) {
6373 switch (VT.SimpleTy) {
6374 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(RetVT, Op0);
6375 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0);
6376 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0);
6377 default: return Register();
6378 }
6379}
6380
6381// FastEmit functions for X86ISD::CVTTS2SIS.
6382
6383Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i32_r(Register Op0) {
6384 if ((Subtarget->hasAVX10_2())) {
6385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SISrr_Int, RC: &X86::GR32RegClass, Op0);
6386 }
6387 return Register();
6388}
6389
6390Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i64_r(Register Op0) {
6391 if ((Subtarget->hasAVX10_2())) {
6392 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6393 }
6394 return Register();
6395}
6396
6397Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6398switch (RetVT.SimpleTy) {
6399 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i32_r(Op0);
6400 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i64_r(Op0);
6401 default: return Register();
6402}
6403}
6404
6405Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i32_r(Register Op0) {
6406 if ((Subtarget->hasAVX10_2())) {
6407 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SISrr_Int, RC: &X86::GR32RegClass, Op0);
6408 }
6409 return Register();
6410}
6411
6412Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i64_r(Register Op0) {
6413 if ((Subtarget->hasAVX10_2())) {
6414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6415 }
6416 return Register();
6417}
6418
6419Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6420switch (RetVT.SimpleTy) {
6421 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i32_r(Op0);
6422 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i64_r(Op0);
6423 default: return Register();
6424}
6425}
6426
6427Register fastEmit_X86ISD_CVTTS2SIS_r(MVT VT, MVT RetVT, Register Op0) {
6428 switch (VT.SimpleTy) {
6429 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_r(RetVT, Op0);
6430 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_r(RetVT, Op0);
6431 default: return Register();
6432 }
6433}
6434
6435// FastEmit functions for X86ISD::CVTTS2SIS_SAE.
6436
6437Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6438 if ((Subtarget->hasAVX10_2())) {
6439 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SISrrb_Int, RC: &X86::GR32RegClass, Op0);
6440 }
6441 return Register();
6442}
6443
6444Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6445 if ((Subtarget->hasAVX10_2())) {
6446 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6447 }
6448 return Register();
6449}
6450
6451Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6452switch (RetVT.SimpleTy) {
6453 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i32_r(Op0);
6454 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i64_r(Op0);
6455 default: return Register();
6456}
6457}
6458
6459Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6460 if ((Subtarget->hasAVX10_2())) {
6461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SISrrb_Int, RC: &X86::GR32RegClass, Op0);
6462 }
6463 return Register();
6464}
6465
6466Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6467 if ((Subtarget->hasAVX10_2())) {
6468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6469 }
6470 return Register();
6471}
6472
6473Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6474switch (RetVT.SimpleTy) {
6475 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i32_r(Op0);
6476 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i64_r(Op0);
6477 default: return Register();
6478}
6479}
6480
6481Register fastEmit_X86ISD_CVTTS2SIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6482 switch (VT.SimpleTy) {
6483 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_r(RetVT, Op0);
6484 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_r(RetVT, Op0);
6485 default: return Register();
6486 }
6487}
6488
6489// FastEmit functions for X86ISD::CVTTS2SI_SAE.
6490
6491Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Register Op0) {
6492 if ((Subtarget->hasFP16())) {
6493 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6494 }
6495 return Register();
6496}
6497
6498Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Register Op0) {
6499 if ((Subtarget->hasFP16())) {
6500 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6501 }
6502 return Register();
6503}
6504
6505Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6506switch (RetVT.SimpleTy) {
6507 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Op0);
6508 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Op0);
6509 default: return Register();
6510}
6511}
6512
6513Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6514 if ((Subtarget->hasAVX512())) {
6515 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6516 }
6517 return Register();
6518}
6519
6520Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6521 if ((Subtarget->hasAVX512())) {
6522 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6523 }
6524 return Register();
6525}
6526
6527Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6528switch (RetVT.SimpleTy) {
6529 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Op0);
6530 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Op0);
6531 default: return Register();
6532}
6533}
6534
6535Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6536 if ((Subtarget->hasAVX512())) {
6537 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6538 }
6539 return Register();
6540}
6541
6542Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6543 if ((Subtarget->hasAVX512())) {
6544 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6545 }
6546 return Register();
6547}
6548
6549Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6550switch (RetVT.SimpleTy) {
6551 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Op0);
6552 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Op0);
6553 default: return Register();
6554}
6555}
6556
6557Register fastEmit_X86ISD_CVTTS2SI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6558 switch (VT.SimpleTy) {
6559 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(RetVT, Op0);
6560 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(RetVT, Op0);
6561 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(RetVT, Op0);
6562 default: return Register();
6563 }
6564}
6565
6566// FastEmit functions for X86ISD::CVTTS2UI.
6567
6568Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Register Op0) {
6569 if ((Subtarget->hasFP16())) {
6570 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6571 }
6572 return Register();
6573}
6574
6575Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Register Op0) {
6576 if ((Subtarget->hasFP16())) {
6577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6578 }
6579 return Register();
6580}
6581
6582Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
6583switch (RetVT.SimpleTy) {
6584 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Op0);
6585 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Op0);
6586 default: return Register();
6587}
6588}
6589
6590Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Register Op0) {
6591 if ((Subtarget->hasAVX512())) {
6592 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6593 }
6594 return Register();
6595}
6596
6597Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Register Op0) {
6598 if ((Subtarget->hasAVX512())) {
6599 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6600 }
6601 return Register();
6602}
6603
6604Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
6605switch (RetVT.SimpleTy) {
6606 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0);
6607 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0);
6608 default: return Register();
6609}
6610}
6611
6612Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Register Op0) {
6613 if ((Subtarget->hasAVX512())) {
6614 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6615 }
6616 return Register();
6617}
6618
6619Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Register Op0) {
6620 if ((Subtarget->hasAVX512())) {
6621 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6622 }
6623 return Register();
6624}
6625
6626Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
6627switch (RetVT.SimpleTy) {
6628 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0);
6629 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0);
6630 default: return Register();
6631}
6632}
6633
6634Register fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, Register Op0) {
6635 switch (VT.SimpleTy) {
6636 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(RetVT, Op0);
6637 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0);
6638 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0);
6639 default: return Register();
6640 }
6641}
6642
6643// FastEmit functions for X86ISD::CVTTS2UIS.
6644
6645Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i32_r(Register Op0) {
6646 if ((Subtarget->hasAVX10_2())) {
6647 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USISrr_Int, RC: &X86::GR32RegClass, Op0);
6648 }
6649 return Register();
6650}
6651
6652Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i64_r(Register Op0) {
6653 if ((Subtarget->hasAVX10_2())) {
6654 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6655 }
6656 return Register();
6657}
6658
6659Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6660switch (RetVT.SimpleTy) {
6661 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i32_r(Op0);
6662 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i64_r(Op0);
6663 default: return Register();
6664}
6665}
6666
6667Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i32_r(Register Op0) {
6668 if ((Subtarget->hasAVX10_2())) {
6669 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USISrr_Int, RC: &X86::GR32RegClass, Op0);
6670 }
6671 return Register();
6672}
6673
6674Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i64_r(Register Op0) {
6675 if ((Subtarget->hasAVX10_2())) {
6676 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6677 }
6678 return Register();
6679}
6680
6681Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6682switch (RetVT.SimpleTy) {
6683 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i32_r(Op0);
6684 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i64_r(Op0);
6685 default: return Register();
6686}
6687}
6688
6689Register fastEmit_X86ISD_CVTTS2UIS_r(MVT VT, MVT RetVT, Register Op0) {
6690 switch (VT.SimpleTy) {
6691 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_r(RetVT, Op0);
6692 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_r(RetVT, Op0);
6693 default: return Register();
6694 }
6695}
6696
6697// FastEmit functions for X86ISD::CVTTS2UIS_SAE.
6698
6699Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6700 if ((Subtarget->hasAVX10_2())) {
6701 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USISrrb_Int, RC: &X86::GR32RegClass, Op0);
6702 }
6703 return Register();
6704}
6705
6706Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6707 if ((Subtarget->hasAVX10_2())) {
6708 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6709 }
6710 return Register();
6711}
6712
6713Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6714switch (RetVT.SimpleTy) {
6715 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i32_r(Op0);
6716 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i64_r(Op0);
6717 default: return Register();
6718}
6719}
6720
6721Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6722 if ((Subtarget->hasAVX10_2())) {
6723 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USISrrb_Int, RC: &X86::GR32RegClass, Op0);
6724 }
6725 return Register();
6726}
6727
6728Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6729 if ((Subtarget->hasAVX10_2())) {
6730 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6731 }
6732 return Register();
6733}
6734
6735Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6736switch (RetVT.SimpleTy) {
6737 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i32_r(Op0);
6738 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i64_r(Op0);
6739 default: return Register();
6740}
6741}
6742
6743Register fastEmit_X86ISD_CVTTS2UIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6744 switch (VT.SimpleTy) {
6745 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_r(RetVT, Op0);
6746 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_r(RetVT, Op0);
6747 default: return Register();
6748 }
6749}
6750
6751// FastEmit functions for X86ISD::CVTTS2UI_SAE.
6752
6753Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Register Op0) {
6754 if ((Subtarget->hasFP16())) {
6755 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6756 }
6757 return Register();
6758}
6759
6760Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Register Op0) {
6761 if ((Subtarget->hasFP16())) {
6762 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6763 }
6764 return Register();
6765}
6766
6767Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6768switch (RetVT.SimpleTy) {
6769 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Op0);
6770 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Op0);
6771 default: return Register();
6772}
6773}
6774
6775Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6776 if ((Subtarget->hasAVX512())) {
6777 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6778 }
6779 return Register();
6780}
6781
6782Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6783 if ((Subtarget->hasAVX512())) {
6784 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6785 }
6786 return Register();
6787}
6788
6789Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6790switch (RetVT.SimpleTy) {
6791 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Op0);
6792 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Op0);
6793 default: return Register();
6794}
6795}
6796
6797Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6798 if ((Subtarget->hasAVX512())) {
6799 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6800 }
6801 return Register();
6802}
6803
6804Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6805 if ((Subtarget->hasAVX512())) {
6806 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6807 }
6808 return Register();
6809}
6810
6811Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6812switch (RetVT.SimpleTy) {
6813 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Op0);
6814 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Op0);
6815 default: return Register();
6816}
6817}
6818
6819Register fastEmit_X86ISD_CVTTS2UI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6820 switch (VT.SimpleTy) {
6821 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(RetVT, Op0);
6822 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(RetVT, Op0);
6823 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(RetVT, Op0);
6824 default: return Register();
6825 }
6826}
6827
6828// FastEmit functions for X86ISD::CVTUI2P.
6829
6830Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
6831 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6832 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6833 }
6834 return Register();
6835}
6836
6837Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
6838 if ((Subtarget->hasVLX())) {
6839 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
6840 }
6841 return Register();
6842}
6843
6844Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
6845switch (RetVT.SimpleTy) {
6846 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
6847 case MVT::v2f64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
6848 default: return Register();
6849}
6850}
6851
6852Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
6853 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6854 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6855 }
6856 return Register();
6857}
6858
6859Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
6860 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6861 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
6862 }
6863 return Register();
6864}
6865
6866Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
6867switch (RetVT.SimpleTy) {
6868 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
6869 case MVT::v4f32: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
6870 default: return Register();
6871}
6872}
6873
6874Register fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
6875 if (RetVT.SimpleTy != MVT::v8f16)
6876 return Register();
6877 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6878 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
6879 }
6880 return Register();
6881}
6882
6883Register fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, Register Op0) {
6884 switch (VT.SimpleTy) {
6885 case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
6886 case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
6887 case MVT::v4i64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
6888 default: return Register();
6889 }
6890}
6891
6892// FastEmit functions for X86ISD::DYN_ALLOCA.
6893
6894Register fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
6895 if (RetVT.SimpleTy != MVT::isVoid)
6896 return Register();
6897 if ((!Subtarget->isTarget64BitLP64())) {
6898 return fastEmitInst_r(MachineInstOpcode: X86::DYN_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
6899 }
6900 return Register();
6901}
6902
6903Register fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
6904 if (RetVT.SimpleTy != MVT::isVoid)
6905 return Register();
6906 if ((Subtarget->is64Bit())) {
6907 return fastEmitInst_r(MachineInstOpcode: X86::DYN_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
6908 }
6909 return Register();
6910}
6911
6912Register fastEmit_X86ISD_DYN_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
6913 switch (VT.SimpleTy) {
6914 case MVT::i32: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(RetVT, Op0);
6915 case MVT::i64: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(RetVT, Op0);
6916 default: return Register();
6917 }
6918}
6919
6920// FastEmit functions for X86ISD::EH_RETURN.
6921
6922Register fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, Register Op0) {
6923 if (RetVT.SimpleTy != MVT::isVoid)
6924 return Register();
6925 return fastEmitInst_r(MachineInstOpcode: X86::EH_RETURN, RC: &X86::GR32RegClass, Op0);
6926}
6927
6928Register fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, Register Op0) {
6929 if (RetVT.SimpleTy != MVT::isVoid)
6930 return Register();
6931 return fastEmitInst_r(MachineInstOpcode: X86::EH_RETURN64, RC: &X86::GR64RegClass, Op0);
6932}
6933
6934Register fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, Register Op0) {
6935 switch (VT.SimpleTy) {
6936 case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0);
6937 case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0);
6938 default: return Register();
6939 }
6940}
6941
6942// FastEmit functions for X86ISD::FGETEXP.
6943
6944Register fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(MVT RetVT, Register Op0) {
6945 if (RetVT.SimpleTy != MVT::v8f16)
6946 return Register();
6947 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6948 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ128r, RC: &X86::VR128XRegClass, Op0);
6949 }
6950 return Register();
6951}
6952
6953Register fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(MVT RetVT, Register Op0) {
6954 if (RetVT.SimpleTy != MVT::v16f16)
6955 return Register();
6956 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6957 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ256r, RC: &X86::VR256XRegClass, Op0);
6958 }
6959 return Register();
6960}
6961
6962Register fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(MVT RetVT, Register Op0) {
6963 if (RetVT.SimpleTy != MVT::v32f16)
6964 return Register();
6965 if ((Subtarget->hasFP16())) {
6966 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZr, RC: &X86::VR512RegClass, Op0);
6967 }
6968 return Register();
6969}
6970
6971Register fastEmit_X86ISD_FGETEXP_MVT_v8bf16_r(MVT RetVT, Register Op0) {
6972 if (RetVT.SimpleTy != MVT::v8bf16)
6973 return Register();
6974 if ((Subtarget->hasAVX10_2())) {
6975 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Z128r, RC: &X86::VR128XRegClass, Op0);
6976 }
6977 return Register();
6978}
6979
6980Register fastEmit_X86ISD_FGETEXP_MVT_v16bf16_r(MVT RetVT, Register Op0) {
6981 if (RetVT.SimpleTy != MVT::v16bf16)
6982 return Register();
6983 if ((Subtarget->hasAVX10_2())) {
6984 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Z256r, RC: &X86::VR256XRegClass, Op0);
6985 }
6986 return Register();
6987}
6988
6989Register fastEmit_X86ISD_FGETEXP_MVT_v32bf16_r(MVT RetVT, Register Op0) {
6990 if (RetVT.SimpleTy != MVT::v32bf16)
6991 return Register();
6992 if ((Subtarget->hasAVX10_2_512())) {
6993 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Zr, RC: &X86::VR512RegClass, Op0);
6994 }
6995 return Register();
6996}
6997
6998Register fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(MVT RetVT, Register Op0) {
6999 if (RetVT.SimpleTy != MVT::v4f32)
7000 return Register();
7001 if ((Subtarget->hasVLX())) {
7002 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ128r, RC: &X86::VR128XRegClass, Op0);
7003 }
7004 return Register();
7005}
7006
7007Register fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7008 if (RetVT.SimpleTy != MVT::v8f32)
7009 return Register();
7010 if ((Subtarget->hasVLX())) {
7011 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ256r, RC: &X86::VR256XRegClass, Op0);
7012 }
7013 return Register();
7014}
7015
7016Register fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7017 if (RetVT.SimpleTy != MVT::v16f32)
7018 return Register();
7019 if ((Subtarget->hasAVX512())) {
7020 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZr, RC: &X86::VR512RegClass, Op0);
7021 }
7022 return Register();
7023}
7024
7025Register fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(MVT RetVT, Register Op0) {
7026 if (RetVT.SimpleTy != MVT::v2f64)
7027 return Register();
7028 if ((Subtarget->hasVLX())) {
7029 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ128r, RC: &X86::VR128XRegClass, Op0);
7030 }
7031 return Register();
7032}
7033
7034Register fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(MVT RetVT, Register Op0) {
7035 if (RetVT.SimpleTy != MVT::v4f64)
7036 return Register();
7037 if ((Subtarget->hasVLX())) {
7038 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ256r, RC: &X86::VR256XRegClass, Op0);
7039 }
7040 return Register();
7041}
7042
7043Register fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(MVT RetVT, Register Op0) {
7044 if (RetVT.SimpleTy != MVT::v8f64)
7045 return Register();
7046 if ((Subtarget->hasAVX512())) {
7047 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZr, RC: &X86::VR512RegClass, Op0);
7048 }
7049 return Register();
7050}
7051
7052Register fastEmit_X86ISD_FGETEXP_r(MVT VT, MVT RetVT, Register Op0) {
7053 switch (VT.SimpleTy) {
7054 case MVT::v8f16: return fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(RetVT, Op0);
7055 case MVT::v16f16: return fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(RetVT, Op0);
7056 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(RetVT, Op0);
7057 case MVT::v8bf16: return fastEmit_X86ISD_FGETEXP_MVT_v8bf16_r(RetVT, Op0);
7058 case MVT::v16bf16: return fastEmit_X86ISD_FGETEXP_MVT_v16bf16_r(RetVT, Op0);
7059 case MVT::v32bf16: return fastEmit_X86ISD_FGETEXP_MVT_v32bf16_r(RetVT, Op0);
7060 case MVT::v4f32: return fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(RetVT, Op0);
7061 case MVT::v8f32: return fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(RetVT, Op0);
7062 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(RetVT, Op0);
7063 case MVT::v2f64: return fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(RetVT, Op0);
7064 case MVT::v4f64: return fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(RetVT, Op0);
7065 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(RetVT, Op0);
7066 default: return Register();
7067 }
7068}
7069
7070// FastEmit functions for X86ISD::FGETEXP_SAE.
7071
7072Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
7073 if (RetVT.SimpleTy != MVT::v32f16)
7074 return Register();
7075 if ((Subtarget->hasFP16())) {
7076 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZrb, RC: &X86::VR512RegClass, Op0);
7077 }
7078 return Register();
7079}
7080
7081Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
7082 if (RetVT.SimpleTy != MVT::v16f32)
7083 return Register();
7084 if ((Subtarget->hasAVX512())) {
7085 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZrb, RC: &X86::VR512RegClass, Op0);
7086 }
7087 return Register();
7088}
7089
7090Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
7091 if (RetVT.SimpleTy != MVT::v8f64)
7092 return Register();
7093 if ((Subtarget->hasAVX512())) {
7094 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZrb, RC: &X86::VR512RegClass, Op0);
7095 }
7096 return Register();
7097}
7098
7099Register fastEmit_X86ISD_FGETEXP_SAE_r(MVT VT, MVT RetVT, Register Op0) {
7100 switch (VT.SimpleTy) {
7101 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(RetVT, Op0);
7102 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(RetVT, Op0);
7103 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(RetVT, Op0);
7104 default: return Register();
7105 }
7106}
7107
7108// FastEmit functions for X86ISD::FP_TO_SINT_SAT.
7109
7110Register fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7111 if (RetVT.SimpleTy != MVT::v2i64)
7112 return Register();
7113 if ((Subtarget->hasAVX10_2())) {
7114 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7115 }
7116 return Register();
7117}
7118
7119Register fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
7120 if (RetVT.SimpleTy != MVT::v4i32)
7121 return Register();
7122 if ((Subtarget->hasAVX10_2())) {
7123 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7124 }
7125 return Register();
7126}
7127
7128Register fastEmit_X86ISD_FP_TO_SINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
7129 switch (VT.SimpleTy) {
7130 case MVT::v4f32: return fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v4f32_r(RetVT, Op0);
7131 case MVT::v2f64: return fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v2f64_r(RetVT, Op0);
7132 default: return Register();
7133 }
7134}
7135
7136// FastEmit functions for X86ISD::FP_TO_UINT_SAT.
7137
7138Register fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7139 if (RetVT.SimpleTy != MVT::v2i64)
7140 return Register();
7141 if ((Subtarget->hasAVX10_2())) {
7142 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7143 }
7144 return Register();
7145}
7146
7147Register fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
7148 if (RetVT.SimpleTy != MVT::v4i32)
7149 return Register();
7150 if ((Subtarget->hasAVX10_2())) {
7151 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7152 }
7153 return Register();
7154}
7155
7156Register fastEmit_X86ISD_FP_TO_UINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
7157 switch (VT.SimpleTy) {
7158 case MVT::v4f32: return fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v4f32_r(RetVT, Op0);
7159 case MVT::v2f64: return fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v2f64_r(RetVT, Op0);
7160 default: return Register();
7161 }
7162}
7163
7164// FastEmit functions for X86ISD::FRCP.
7165
7166Register fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, Register Op0) {
7167 if (RetVT.SimpleTy != MVT::f32)
7168 return Register();
7169 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7170 return fastEmitInst_r(MachineInstOpcode: X86::RCPSSr, RC: &X86::FR32RegClass, Op0);
7171 }
7172 return Register();
7173}
7174
7175Register fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7176 if (RetVT.SimpleTy != MVT::v4f32)
7177 return Register();
7178 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7179 return fastEmitInst_r(MachineInstOpcode: X86::RCPPSr, RC: &X86::VR128RegClass, Op0);
7180 }
7181 if ((Subtarget->hasAVX())) {
7182 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSr, RC: &X86::VR128RegClass, Op0);
7183 }
7184 return Register();
7185}
7186
7187Register fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7188 if (RetVT.SimpleTy != MVT::v8f32)
7189 return Register();
7190 if ((Subtarget->hasAVX())) {
7191 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSYr, RC: &X86::VR256RegClass, Op0);
7192 }
7193 return Register();
7194}
7195
7196Register fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, Register Op0) {
7197 switch (VT.SimpleTy) {
7198 case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0);
7199 case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0);
7200 case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0);
7201 default: return Register();
7202 }
7203}
7204
7205// FastEmit functions for X86ISD::FRSQRT.
7206
7207Register fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
7208 if (RetVT.SimpleTy != MVT::f32)
7209 return Register();
7210 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7211 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTSSr, RC: &X86::FR32RegClass, Op0);
7212 }
7213 return Register();
7214}
7215
7216Register fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7217 if (RetVT.SimpleTy != MVT::v4f32)
7218 return Register();
7219 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7220 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTPSr, RC: &X86::VR128RegClass, Op0);
7221 }
7222 if ((Subtarget->hasAVX())) {
7223 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSr, RC: &X86::VR128RegClass, Op0);
7224 }
7225 return Register();
7226}
7227
7228Register fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
7229 if (RetVT.SimpleTy != MVT::v8f32)
7230 return Register();
7231 if ((Subtarget->hasAVX())) {
7232 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSYr, RC: &X86::VR256RegClass, Op0);
7233 }
7234 return Register();
7235}
7236
7237Register fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, Register Op0) {
7238 switch (VT.SimpleTy) {
7239 case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0);
7240 case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0);
7241 case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0);
7242 default: return Register();
7243 }
7244}
7245
7246// FastEmit functions for X86ISD::MMX_MOVD2W.
7247
7248Register fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, Register Op0) {
7249 if (RetVT.SimpleTy != MVT::i32)
7250 return Register();
7251 if ((Subtarget->hasMMX())) {
7252 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64grr, RC: &X86::GR32RegClass, Op0);
7253 }
7254 return Register();
7255}
7256
7257Register fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, Register Op0) {
7258 switch (VT.SimpleTy) {
7259 case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0);
7260 default: return Register();
7261 }
7262}
7263
7264// FastEmit functions for X86ISD::MMX_MOVW2D.
7265
7266Register fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, Register Op0) {
7267 if (RetVT.SimpleTy != MVT::x86mmx)
7268 return Register();
7269 if ((Subtarget->hasMMX())) {
7270 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64rr, RC: &X86::VR64RegClass, Op0);
7271 }
7272 return Register();
7273}
7274
7275Register fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, Register Op0) {
7276 switch (VT.SimpleTy) {
7277 case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0);
7278 default: return Register();
7279 }
7280}
7281
7282// FastEmit functions for X86ISD::MOVDDUP.
7283
7284Register fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, Register Op0) {
7285 if (RetVT.SimpleTy != MVT::v2f64)
7286 return Register();
7287 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7288 return fastEmitInst_r(MachineInstOpcode: X86::MOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7289 }
7290 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7291 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7292 }
7293 return Register();
7294}
7295
7296Register fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, Register Op0) {
7297 if (RetVT.SimpleTy != MVT::v4f64)
7298 return Register();
7299 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7300 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7301 }
7302 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7303 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPYrr, RC: &X86::VR256RegClass, Op0);
7304 }
7305 return Register();
7306}
7307
7308Register fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, Register Op0) {
7309 if (RetVT.SimpleTy != MVT::v8f64)
7310 return Register();
7311 if ((Subtarget->hasAVX512())) {
7312 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZrr, RC: &X86::VR512RegClass, Op0);
7313 }
7314 return Register();
7315}
7316
7317Register fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, Register Op0) {
7318 switch (VT.SimpleTy) {
7319 case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0);
7320 case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0);
7321 case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0);
7322 default: return Register();
7323 }
7324}
7325
7326// FastEmit functions for X86ISD::MOVDQ2Q.
7327
7328Register fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, Register Op0) {
7329 if (RetVT.SimpleTy != MVT::x86mmx)
7330 return Register();
7331 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
7332 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVDQ2Qrr, RC: &X86::VR64RegClass, Op0);
7333 }
7334 return Register();
7335}
7336
7337Register fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, Register Op0) {
7338 switch (VT.SimpleTy) {
7339 case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0);
7340 default: return Register();
7341 }
7342}
7343
7344// FastEmit functions for X86ISD::MOVMSK.
7345
7346Register fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, Register Op0) {
7347 if (RetVT.SimpleTy != MVT::i32)
7348 return Register();
7349 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7350 return fastEmitInst_r(MachineInstOpcode: X86::PMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
7351 }
7352 if ((Subtarget->hasAVX())) {
7353 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
7354 }
7355 return Register();
7356}
7357
7358Register fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, Register Op0) {
7359 if (RetVT.SimpleTy != MVT::i32)
7360 return Register();
7361 if ((Subtarget->hasAVX2())) {
7362 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBYrr, RC: &X86::GR32RegClass, Op0);
7363 }
7364 return Register();
7365}
7366
7367Register fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, Register Op0) {
7368 if (RetVT.SimpleTy != MVT::i32)
7369 return Register();
7370 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7371 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7372 }
7373 if ((Subtarget->hasAVX())) {
7374 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7375 }
7376 return Register();
7377}
7378
7379Register fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, Register Op0) {
7380 if (RetVT.SimpleTy != MVT::i32)
7381 return Register();
7382 if ((Subtarget->hasAVX())) {
7383 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
7384 }
7385 return Register();
7386}
7387
7388Register fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, Register Op0) {
7389 if (RetVT.SimpleTy != MVT::i32)
7390 return Register();
7391 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7392 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7393 }
7394 if ((Subtarget->hasAVX())) {
7395 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7396 }
7397 return Register();
7398}
7399
7400Register fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, Register Op0) {
7401 if (RetVT.SimpleTy != MVT::i32)
7402 return Register();
7403 if ((Subtarget->hasAVX())) {
7404 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
7405 }
7406 return Register();
7407}
7408
7409Register fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, Register Op0) {
7410 if (RetVT.SimpleTy != MVT::i32)
7411 return Register();
7412 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7413 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7414 }
7415 if ((Subtarget->hasAVX())) {
7416 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7417 }
7418 return Register();
7419}
7420
7421Register fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, Register Op0) {
7422 if (RetVT.SimpleTy != MVT::i32)
7423 return Register();
7424 if ((Subtarget->hasAVX())) {
7425 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
7426 }
7427 return Register();
7428}
7429
7430Register fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, Register Op0) {
7431 if (RetVT.SimpleTy != MVT::i32)
7432 return Register();
7433 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7434 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7435 }
7436 if ((Subtarget->hasAVX())) {
7437 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7438 }
7439 return Register();
7440}
7441
7442Register fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, Register Op0) {
7443 if (RetVT.SimpleTy != MVT::i32)
7444 return Register();
7445 if ((Subtarget->hasAVX())) {
7446 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
7447 }
7448 return Register();
7449}
7450
7451Register fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, Register Op0) {
7452 switch (VT.SimpleTy) {
7453 case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0);
7454 case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0);
7455 case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0);
7456 case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0);
7457 case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0);
7458 case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0);
7459 case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0);
7460 case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0);
7461 case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0);
7462 case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0);
7463 default: return Register();
7464 }
7465}
7466
7467// FastEmit functions for X86ISD::MOVQ2DQ.
7468
7469Register fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(MVT RetVT, Register Op0) {
7470 if (RetVT.SimpleTy != MVT::v2i64)
7471 return Register();
7472 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
7473 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2DQrr, RC: &X86::VR128RegClass, Op0);
7474 }
7475 return Register();
7476}
7477
7478Register fastEmit_X86ISD_MOVQ2DQ_r(MVT VT, MVT RetVT, Register Op0) {
7479 switch (VT.SimpleTy) {
7480 case MVT::x86mmx: return fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(RetVT, Op0);
7481 default: return Register();
7482 }
7483}
7484
7485// FastEmit functions for X86ISD::MOVSHDUP.
7486
7487Register fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, Register Op0) {
7488 if (RetVT.SimpleTy != MVT::v4i32)
7489 return Register();
7490 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7491 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7492 }
7493 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7494 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7495 }
7496 return Register();
7497}
7498
7499Register fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, Register Op0) {
7500 if (RetVT.SimpleTy != MVT::v8i32)
7501 return Register();
7502 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7503 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
7504 }
7505 return Register();
7506}
7507
7508Register fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7509 if (RetVT.SimpleTy != MVT::v4f32)
7510 return Register();
7511 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7512 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7513 }
7514 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7515 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7516 }
7517 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7518 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7519 }
7520 return Register();
7521}
7522
7523Register fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7524 if (RetVT.SimpleTy != MVT::v8f32)
7525 return Register();
7526 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7527 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7528 }
7529 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7530 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
7531 }
7532 return Register();
7533}
7534
7535Register fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7536 if (RetVT.SimpleTy != MVT::v16f32)
7537 return Register();
7538 if ((Subtarget->hasAVX512())) {
7539 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZrr, RC: &X86::VR512RegClass, Op0);
7540 }
7541 return Register();
7542}
7543
7544Register fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, Register Op0) {
7545 switch (VT.SimpleTy) {
7546 case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0);
7547 case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0);
7548 case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0);
7549 case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0);
7550 case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0);
7551 default: return Register();
7552 }
7553}
7554
7555// FastEmit functions for X86ISD::MOVSLDUP.
7556
7557Register fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, Register Op0) {
7558 if (RetVT.SimpleTy != MVT::v4i32)
7559 return Register();
7560 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7561 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7562 }
7563 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7564 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7565 }
7566 return Register();
7567}
7568
7569Register fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, Register Op0) {
7570 if (RetVT.SimpleTy != MVT::v8i32)
7571 return Register();
7572 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7573 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
7574 }
7575 return Register();
7576}
7577
7578Register fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7579 if (RetVT.SimpleTy != MVT::v4f32)
7580 return Register();
7581 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7582 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7583 }
7584 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7585 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7586 }
7587 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7588 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7589 }
7590 return Register();
7591}
7592
7593Register fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7594 if (RetVT.SimpleTy != MVT::v8f32)
7595 return Register();
7596 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7597 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7598 }
7599 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7600 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
7601 }
7602 return Register();
7603}
7604
7605Register fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7606 if (RetVT.SimpleTy != MVT::v16f32)
7607 return Register();
7608 if ((Subtarget->hasAVX512())) {
7609 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZrr, RC: &X86::VR512RegClass, Op0);
7610 }
7611 return Register();
7612}
7613
7614Register fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, Register Op0) {
7615 switch (VT.SimpleTy) {
7616 case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0);
7617 case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0);
7618 case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0);
7619 case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0);
7620 case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0);
7621 default: return Register();
7622 }
7623}
7624
7625// FastEmit functions for X86ISD::NT_BRIND.
7626
7627Register fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, Register Op0) {
7628 if (RetVT.SimpleTy != MVT::isVoid)
7629 return Register();
7630 if ((!Subtarget->is64Bit())) {
7631 return fastEmitInst_r(MachineInstOpcode: X86::JMP16r_NT, RC: &X86::GR16RegClass, Op0);
7632 }
7633 return Register();
7634}
7635
7636Register fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
7637 if (RetVT.SimpleTy != MVT::isVoid)
7638 return Register();
7639 if ((!Subtarget->is64Bit())) {
7640 return fastEmitInst_r(MachineInstOpcode: X86::JMP32r_NT, RC: &X86::GR32RegClass, Op0);
7641 }
7642 return Register();
7643}
7644
7645Register fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
7646 if (RetVT.SimpleTy != MVT::isVoid)
7647 return Register();
7648 if ((Subtarget->is64Bit())) {
7649 return fastEmitInst_r(MachineInstOpcode: X86::JMP64r_NT, RC: &X86::GR64RegClass, Op0);
7650 }
7651 return Register();
7652}
7653
7654Register fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
7655 switch (VT.SimpleTy) {
7656 case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0);
7657 case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0);
7658 case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0);
7659 default: return Register();
7660 }
7661}
7662
7663// FastEmit functions for X86ISD::NT_CALL.
7664
7665Register fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, Register Op0) {
7666 if (RetVT.SimpleTy != MVT::isVoid)
7667 return Register();
7668 if ((!Subtarget->is64Bit())) {
7669 return fastEmitInst_r(MachineInstOpcode: X86::CALL16r_NT, RC: &X86::GR16RegClass, Op0);
7670 }
7671 return Register();
7672}
7673
7674Register fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
7675 if (RetVT.SimpleTy != MVT::isVoid)
7676 return Register();
7677 if ((!Subtarget->is64Bit())) {
7678 return fastEmitInst_r(MachineInstOpcode: X86::CALL32r_NT, RC: &X86::GR32RegClass, Op0);
7679 }
7680 return Register();
7681}
7682
7683Register fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
7684 if (RetVT.SimpleTy != MVT::isVoid)
7685 return Register();
7686 if ((Subtarget->is64Bit())) {
7687 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r_NT, RC: &X86::GR64RegClass, Op0);
7688 }
7689 return Register();
7690}
7691
7692Register fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, Register Op0) {
7693 switch (VT.SimpleTy) {
7694 case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0);
7695 case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0);
7696 case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0);
7697 default: return Register();
7698 }
7699}
7700
7701// FastEmit functions for X86ISD::PHMINPOS.
7702
7703Register fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, Register Op0) {
7704 if (RetVT.SimpleTy != MVT::v8i16)
7705 return Register();
7706 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
7707 return fastEmitInst_r(MachineInstOpcode: X86::PHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
7708 }
7709 if ((Subtarget->hasAVX())) {
7710 return fastEmitInst_r(MachineInstOpcode: X86::VPHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
7711 }
7712 return Register();
7713}
7714
7715Register fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, Register Op0) {
7716 switch (VT.SimpleTy) {
7717 case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0);
7718 default: return Register();
7719 }
7720}
7721
7722// FastEmit functions for X86ISD::PROBED_ALLOCA.
7723
7724Register fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
7725 if (RetVT.SimpleTy != MVT::i32)
7726 return Register();
7727 if ((!Subtarget->isTarget64BitLP64())) {
7728 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
7729 }
7730 return Register();
7731}
7732
7733Register fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
7734 if (RetVT.SimpleTy != MVT::i64)
7735 return Register();
7736 if ((Subtarget->is64Bit())) {
7737 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
7738 }
7739 return Register();
7740}
7741
7742Register fastEmit_X86ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
7743 switch (VT.SimpleTy) {
7744 case MVT::i32: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(RetVT, Op0);
7745 case MVT::i64: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
7746 default: return Register();
7747 }
7748}
7749
7750// FastEmit functions for X86ISD::RCP14.
7751
7752Register fastEmit_X86ISD_RCP14_MVT_v8f16_r(MVT RetVT, Register Op0) {
7753 if (RetVT.SimpleTy != MVT::v8f16)
7754 return Register();
7755 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7756 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ128r, RC: &X86::VR128XRegClass, Op0);
7757 }
7758 return Register();
7759}
7760
7761Register fastEmit_X86ISD_RCP14_MVT_v16f16_r(MVT RetVT, Register Op0) {
7762 if (RetVT.SimpleTy != MVT::v16f16)
7763 return Register();
7764 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7765 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ256r, RC: &X86::VR256XRegClass, Op0);
7766 }
7767 return Register();
7768}
7769
7770Register fastEmit_X86ISD_RCP14_MVT_v32f16_r(MVT RetVT, Register Op0) {
7771 if (RetVT.SimpleTy != MVT::v32f16)
7772 return Register();
7773 if ((Subtarget->hasFP16())) {
7774 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZr, RC: &X86::VR512RegClass, Op0);
7775 }
7776 return Register();
7777}
7778
7779Register fastEmit_X86ISD_RCP14_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7780 if (RetVT.SimpleTy != MVT::v8bf16)
7781 return Register();
7782 if ((Subtarget->hasAVX10_2())) {
7783 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7784 }
7785 return Register();
7786}
7787
7788Register fastEmit_X86ISD_RCP14_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7789 if (RetVT.SimpleTy != MVT::v16bf16)
7790 return Register();
7791 if ((Subtarget->hasAVX10_2())) {
7792 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7793 }
7794 return Register();
7795}
7796
7797Register fastEmit_X86ISD_RCP14_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7798 if (RetVT.SimpleTy != MVT::v32bf16)
7799 return Register();
7800 if ((Subtarget->hasAVX10_2_512())) {
7801 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Zr, RC: &X86::VR512RegClass, Op0);
7802 }
7803 return Register();
7804}
7805
7806Register fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, Register Op0) {
7807 if (RetVT.SimpleTy != MVT::v4f32)
7808 return Register();
7809 if ((Subtarget->hasVLX())) {
7810 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ128r, RC: &X86::VR128XRegClass, Op0);
7811 }
7812 return Register();
7813}
7814
7815Register fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, Register Op0) {
7816 if (RetVT.SimpleTy != MVT::v8f32)
7817 return Register();
7818 if ((Subtarget->hasVLX())) {
7819 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ256r, RC: &X86::VR256XRegClass, Op0);
7820 }
7821 return Register();
7822}
7823
7824Register fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, Register Op0) {
7825 if (RetVT.SimpleTy != MVT::v16f32)
7826 return Register();
7827 if ((Subtarget->hasAVX512())) {
7828 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZr, RC: &X86::VR512RegClass, Op0);
7829 }
7830 return Register();
7831}
7832
7833Register fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, Register Op0) {
7834 if (RetVT.SimpleTy != MVT::v2f64)
7835 return Register();
7836 if ((Subtarget->hasVLX())) {
7837 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ128r, RC: &X86::VR128XRegClass, Op0);
7838 }
7839 return Register();
7840}
7841
7842Register fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, Register Op0) {
7843 if (RetVT.SimpleTy != MVT::v4f64)
7844 return Register();
7845 if ((Subtarget->hasVLX())) {
7846 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ256r, RC: &X86::VR256XRegClass, Op0);
7847 }
7848 return Register();
7849}
7850
7851Register fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, Register Op0) {
7852 if (RetVT.SimpleTy != MVT::v8f64)
7853 return Register();
7854 if ((Subtarget->hasAVX512())) {
7855 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZr, RC: &X86::VR512RegClass, Op0);
7856 }
7857 return Register();
7858}
7859
7860Register fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, Register Op0) {
7861 switch (VT.SimpleTy) {
7862 case MVT::v8f16: return fastEmit_X86ISD_RCP14_MVT_v8f16_r(RetVT, Op0);
7863 case MVT::v16f16: return fastEmit_X86ISD_RCP14_MVT_v16f16_r(RetVT, Op0);
7864 case MVT::v32f16: return fastEmit_X86ISD_RCP14_MVT_v32f16_r(RetVT, Op0);
7865 case MVT::v8bf16: return fastEmit_X86ISD_RCP14_MVT_v8bf16_r(RetVT, Op0);
7866 case MVT::v16bf16: return fastEmit_X86ISD_RCP14_MVT_v16bf16_r(RetVT, Op0);
7867 case MVT::v32bf16: return fastEmit_X86ISD_RCP14_MVT_v32bf16_r(RetVT, Op0);
7868 case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0);
7869 case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0);
7870 case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0);
7871 case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0);
7872 case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0);
7873 case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0);
7874 default: return Register();
7875 }
7876}
7877
7878// FastEmit functions for X86ISD::RSQRT14.
7879
7880Register fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(MVT RetVT, Register Op0) {
7881 if (RetVT.SimpleTy != MVT::v8f16)
7882 return Register();
7883 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7884 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
7885 }
7886 return Register();
7887}
7888
7889Register fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(MVT RetVT, Register Op0) {
7890 if (RetVT.SimpleTy != MVT::v16f16)
7891 return Register();
7892 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7893 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
7894 }
7895 return Register();
7896}
7897
7898Register fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(MVT RetVT, Register Op0) {
7899 if (RetVT.SimpleTy != MVT::v32f16)
7900 return Register();
7901 if ((Subtarget->hasFP16())) {
7902 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZr, RC: &X86::VR512RegClass, Op0);
7903 }
7904 return Register();
7905}
7906
7907Register fastEmit_X86ISD_RSQRT14_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7908 if (RetVT.SimpleTy != MVT::v8bf16)
7909 return Register();
7910 if ((Subtarget->hasAVX10_2())) {
7911 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7912 }
7913 return Register();
7914}
7915
7916Register fastEmit_X86ISD_RSQRT14_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7917 if (RetVT.SimpleTy != MVT::v16bf16)
7918 return Register();
7919 if ((Subtarget->hasAVX10_2())) {
7920 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7921 }
7922 return Register();
7923}
7924
7925Register fastEmit_X86ISD_RSQRT14_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7926 if (RetVT.SimpleTy != MVT::v32bf16)
7927 return Register();
7928 if ((Subtarget->hasAVX10_2_512())) {
7929 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
7930 }
7931 return Register();
7932}
7933
7934Register fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, Register Op0) {
7935 if (RetVT.SimpleTy != MVT::v4f32)
7936 return Register();
7937 if ((Subtarget->hasVLX())) {
7938 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ128r, RC: &X86::VR128XRegClass, Op0);
7939 }
7940 return Register();
7941}
7942
7943Register fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, Register Op0) {
7944 if (RetVT.SimpleTy != MVT::v8f32)
7945 return Register();
7946 if ((Subtarget->hasVLX())) {
7947 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ256r, RC: &X86::VR256XRegClass, Op0);
7948 }
7949 return Register();
7950}
7951
7952Register fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, Register Op0) {
7953 if (RetVT.SimpleTy != MVT::v16f32)
7954 return Register();
7955 if ((Subtarget->hasAVX512())) {
7956 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZr, RC: &X86::VR512RegClass, Op0);
7957 }
7958 return Register();
7959}
7960
7961Register fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, Register Op0) {
7962 if (RetVT.SimpleTy != MVT::v2f64)
7963 return Register();
7964 if ((Subtarget->hasVLX())) {
7965 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ128r, RC: &X86::VR128XRegClass, Op0);
7966 }
7967 return Register();
7968}
7969
7970Register fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, Register Op0) {
7971 if (RetVT.SimpleTy != MVT::v4f64)
7972 return Register();
7973 if ((Subtarget->hasVLX())) {
7974 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ256r, RC: &X86::VR256XRegClass, Op0);
7975 }
7976 return Register();
7977}
7978
7979Register fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, Register Op0) {
7980 if (RetVT.SimpleTy != MVT::v8f64)
7981 return Register();
7982 if ((Subtarget->hasAVX512())) {
7983 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZr, RC: &X86::VR512RegClass, Op0);
7984 }
7985 return Register();
7986}
7987
7988Register fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, Register Op0) {
7989 switch (VT.SimpleTy) {
7990 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(RetVT, Op0);
7991 case MVT::v16f16: return fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(RetVT, Op0);
7992 case MVT::v32f16: return fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(RetVT, Op0);
7993 case MVT::v8bf16: return fastEmit_X86ISD_RSQRT14_MVT_v8bf16_r(RetVT, Op0);
7994 case MVT::v16bf16: return fastEmit_X86ISD_RSQRT14_MVT_v16bf16_r(RetVT, Op0);
7995 case MVT::v32bf16: return fastEmit_X86ISD_RSQRT14_MVT_v32bf16_r(RetVT, Op0);
7996 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0);
7997 case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0);
7998 case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0);
7999 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0);
8000 case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0);
8001 case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0);
8002 default: return Register();
8003 }
8004}
8005
8006// FastEmit functions for X86ISD::SEG_ALLOCA.
8007
8008Register fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
8009 if (RetVT.SimpleTy != MVT::i32)
8010 return Register();
8011 if ((!Subtarget->isTarget64BitLP64())) {
8012 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
8013 }
8014 return Register();
8015}
8016
8017Register fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
8018 if (RetVT.SimpleTy != MVT::i64)
8019 return Register();
8020 if ((Subtarget->is64Bit())) {
8021 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
8022 }
8023 return Register();
8024}
8025
8026Register fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
8027 switch (VT.SimpleTy) {
8028 case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0);
8029 case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0);
8030 default: return Register();
8031 }
8032}
8033
8034// FastEmit functions for X86ISD::STRICT_CVTPH2PS.
8035
8036Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Register Op0) {
8037 if ((Subtarget->hasVLX())) {
8038 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8039 }
8040 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
8041 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
8042 }
8043 return Register();
8044}
8045
8046Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Register Op0) {
8047 if ((Subtarget->hasVLX())) {
8048 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
8049 }
8050 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
8051 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
8052 }
8053 return Register();
8054}
8055
8056Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(MVT RetVT, Register Op0) {
8057switch (RetVT.SimpleTy) {
8058 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
8059 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
8060 default: return Register();
8061}
8062}
8063
8064Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(MVT RetVT, Register Op0) {
8065 if (RetVT.SimpleTy != MVT::v16f32)
8066 return Register();
8067 if ((Subtarget->hasAVX512())) {
8068 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
8069 }
8070 return Register();
8071}
8072
8073Register fastEmit_X86ISD_STRICT_CVTPH2PS_r(MVT VT, MVT RetVT, Register Op0) {
8074 switch (VT.SimpleTy) {
8075 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
8076 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
8077 default: return Register();
8078 }
8079}
8080
8081// FastEmit functions for X86ISD::STRICT_CVTSI2P.
8082
8083Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
8084 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8085 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8086 }
8087 return Register();
8088}
8089
8090Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
8091 if ((Subtarget->hasVLX())) {
8092 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8093 }
8094 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8095 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
8096 }
8097 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8098 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
8099 }
8100 return Register();
8101}
8102
8103Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
8104switch (RetVT.SimpleTy) {
8105 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
8106 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
8107 default: return Register();
8108}
8109}
8110
8111Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
8112 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8113 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8114 }
8115 return Register();
8116}
8117
8118Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
8119 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8120 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8121 }
8122 return Register();
8123}
8124
8125Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
8126switch (RetVT.SimpleTy) {
8127 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
8128 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
8129 default: return Register();
8130}
8131}
8132
8133Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
8134 if (RetVT.SimpleTy != MVT::v8f16)
8135 return Register();
8136 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8137 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8138 }
8139 return Register();
8140}
8141
8142Register fastEmit_X86ISD_STRICT_CVTSI2P_r(MVT VT, MVT RetVT, Register Op0) {
8143 switch (VT.SimpleTy) {
8144 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
8145 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
8146 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
8147 default: return Register();
8148 }
8149}
8150
8151// FastEmit functions for X86ISD::STRICT_CVTTP2SI.
8152
8153Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
8154 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8155 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
8156 }
8157 return Register();
8158}
8159
8160Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
8161 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8162 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8163 }
8164 return Register();
8165}
8166
8167Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
8168 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8169 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
8170 }
8171 return Register();
8172}
8173
8174Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
8175 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8176 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8177 }
8178 return Register();
8179}
8180
8181Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
8182 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8183 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8184 }
8185 return Register();
8186}
8187
8188Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
8189 if ((Subtarget->hasFP16())) {
8190 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
8191 }
8192 return Register();
8193}
8194
8195Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
8196switch (RetVT.SimpleTy) {
8197 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
8198 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
8199 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
8200 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
8201 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
8202 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
8203 default: return Register();
8204}
8205}
8206
8207Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
8208 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
8210 }
8211 return Register();
8212}
8213
8214Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
8215 if ((Subtarget->hasFP16())) {
8216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
8217 }
8218 return Register();
8219}
8220
8221Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
8222switch (RetVT.SimpleTy) {
8223 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
8224 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
8225 default: return Register();
8226}
8227}
8228
8229Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
8230 if (RetVT.SimpleTy != MVT::v32i16)
8231 return Register();
8232 if ((Subtarget->hasFP16())) {
8233 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
8234 }
8235 return Register();
8236}
8237
8238Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
8239 if ((Subtarget->hasVLX())) {
8240 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8241 }
8242 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8243 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
8244 }
8245 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8246 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
8247 }
8248 return Register();
8249}
8250
8251Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
8252 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8253 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8254 }
8255 return Register();
8256}
8257
8258Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
8259 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8260 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8261 }
8262 return Register();
8263}
8264
8265Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
8266switch (RetVT.SimpleTy) {
8267 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
8268 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
8269 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
8270 default: return Register();
8271}
8272}
8273
8274Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
8275 if ((Subtarget->hasVLX())) {
8276 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
8277 }
8278 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8279 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
8280 }
8281 return Register();
8282}
8283
8284Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
8285 if ((Subtarget->hasDQI())) {
8286 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
8287 }
8288 return Register();
8289}
8290
8291Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
8292switch (RetVT.SimpleTy) {
8293 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
8294 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
8295 default: return Register();
8296}
8297}
8298
8299Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
8300 if (RetVT.SimpleTy != MVT::v16i32)
8301 return Register();
8302 if ((Subtarget->hasAVX512())) {
8303 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
8304 }
8305 return Register();
8306}
8307
8308Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
8309 if ((Subtarget->hasVLX())) {
8310 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8311 }
8312 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8313 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
8314 }
8315 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8316 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
8317 }
8318 return Register();
8319}
8320
8321Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
8322 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8323 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8324 }
8325 return Register();
8326}
8327
8328Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
8329switch (RetVT.SimpleTy) {
8330 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
8331 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
8332 default: return Register();
8333}
8334}
8335
8336Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
8337 if ((Subtarget->hasVLX())) {
8338 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
8339 }
8340 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8341 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
8342 }
8343 return Register();
8344}
8345
8346Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
8347 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8349 }
8350 return Register();
8351}
8352
8353Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
8354switch (RetVT.SimpleTy) {
8355 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
8356 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
8357 default: return Register();
8358}
8359}
8360
8361Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
8362 if ((Subtarget->hasAVX512())) {
8363 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
8364 }
8365 return Register();
8366}
8367
8368Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
8369 if ((Subtarget->hasDQI())) {
8370 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
8371 }
8372 return Register();
8373}
8374
8375Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
8376switch (RetVT.SimpleTy) {
8377 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
8378 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
8379 default: return Register();
8380}
8381}
8382
8383Register fastEmit_X86ISD_STRICT_CVTTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
8384 switch (VT.SimpleTy) {
8385 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
8386 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
8387 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
8388 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
8389 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
8390 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
8391 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
8392 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
8393 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
8394 default: return Register();
8395 }
8396}
8397
8398// FastEmit functions for X86ISD::STRICT_CVTTP2UI.
8399
8400Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
8401 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8402 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
8403 }
8404 return Register();
8405}
8406
8407Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
8408 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8409 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8410 }
8411 return Register();
8412}
8413
8414Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
8415 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8416 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
8417 }
8418 return Register();
8419}
8420
8421Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
8422 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8423 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8424 }
8425 return Register();
8426}
8427
8428Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
8429 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8430 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8431 }
8432 return Register();
8433}
8434
8435Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
8436 if ((Subtarget->hasFP16())) {
8437 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
8438 }
8439 return Register();
8440}
8441
8442Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
8443switch (RetVT.SimpleTy) {
8444 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
8445 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
8446 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
8447 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
8448 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
8449 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
8450 default: return Register();
8451}
8452}
8453
8454Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
8455 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8456 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
8457 }
8458 return Register();
8459}
8460
8461Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
8462 if ((Subtarget->hasFP16())) {
8463 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
8464 }
8465 return Register();
8466}
8467
8468Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
8469switch (RetVT.SimpleTy) {
8470 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
8471 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
8472 default: return Register();
8473}
8474}
8475
8476Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
8477 if (RetVT.SimpleTy != MVT::v32i16)
8478 return Register();
8479 if ((Subtarget->hasFP16())) {
8480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
8481 }
8482 return Register();
8483}
8484
8485Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
8486 if ((Subtarget->hasVLX())) {
8487 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8488 }
8489 return Register();
8490}
8491
8492Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
8493 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8495 }
8496 return Register();
8497}
8498
8499Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
8500 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8502 }
8503 return Register();
8504}
8505
8506Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
8507switch (RetVT.SimpleTy) {
8508 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
8509 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
8510 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
8511 default: return Register();
8512}
8513}
8514
8515Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
8516 if ((Subtarget->hasVLX())) {
8517 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
8518 }
8519 return Register();
8520}
8521
8522Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
8523 if ((Subtarget->hasDQI())) {
8524 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
8525 }
8526 return Register();
8527}
8528
8529Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
8530switch (RetVT.SimpleTy) {
8531 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
8532 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
8533 default: return Register();
8534}
8535}
8536
8537Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
8538 if (RetVT.SimpleTy != MVT::v16i32)
8539 return Register();
8540 if ((Subtarget->hasAVX512())) {
8541 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
8542 }
8543 return Register();
8544}
8545
8546Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
8547 if ((Subtarget->hasVLX())) {
8548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8549 }
8550 return Register();
8551}
8552
8553Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
8554 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8556 }
8557 return Register();
8558}
8559
8560Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
8561switch (RetVT.SimpleTy) {
8562 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
8563 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
8564 default: return Register();
8565}
8566}
8567
8568Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
8569 if ((Subtarget->hasVLX())) {
8570 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
8571 }
8572 return Register();
8573}
8574
8575Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
8576 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8578 }
8579 return Register();
8580}
8581
8582Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
8583switch (RetVT.SimpleTy) {
8584 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
8585 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
8586 default: return Register();
8587}
8588}
8589
8590Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
8591 if ((Subtarget->hasAVX512())) {
8592 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
8593 }
8594 return Register();
8595}
8596
8597Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
8598 if ((Subtarget->hasDQI())) {
8599 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
8600 }
8601 return Register();
8602}
8603
8604Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
8605switch (RetVT.SimpleTy) {
8606 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
8607 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
8608 default: return Register();
8609}
8610}
8611
8612Register fastEmit_X86ISD_STRICT_CVTTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
8613 switch (VT.SimpleTy) {
8614 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
8615 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
8616 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
8617 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
8618 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
8619 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
8620 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
8621 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
8622 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
8623 default: return Register();
8624 }
8625}
8626
8627// FastEmit functions for X86ISD::STRICT_CVTUI2P.
8628
8629Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
8630 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8631 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8632 }
8633 return Register();
8634}
8635
8636Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
8637 if ((Subtarget->hasVLX())) {
8638 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8639 }
8640 return Register();
8641}
8642
8643Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
8644switch (RetVT.SimpleTy) {
8645 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
8646 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
8647 default: return Register();
8648}
8649}
8650
8651Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
8652 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8653 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8654 }
8655 return Register();
8656}
8657
8658Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
8659 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8660 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8661 }
8662 return Register();
8663}
8664
8665Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
8666switch (RetVT.SimpleTy) {
8667 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
8668 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
8669 default: return Register();
8670}
8671}
8672
8673Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
8674 if (RetVT.SimpleTy != MVT::v8f16)
8675 return Register();
8676 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8677 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8678 }
8679 return Register();
8680}
8681
8682Register fastEmit_X86ISD_STRICT_CVTUI2P_r(MVT VT, MVT RetVT, Register Op0) {
8683 switch (VT.SimpleTy) {
8684 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
8685 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
8686 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
8687 default: return Register();
8688 }
8689}
8690
8691// FastEmit functions for X86ISD::STRICT_VFPEXT.
8692
8693Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Register Op0) {
8694 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8695 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
8696 }
8697 return Register();
8698}
8699
8700Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Register Op0) {
8701 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8702 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8703 }
8704 return Register();
8705}
8706
8707Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Register Op0) {
8708 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8709 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
8710 }
8711 return Register();
8712}
8713
8714Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(MVT RetVT, Register Op0) {
8715switch (RetVT.SimpleTy) {
8716 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
8717 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
8718 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
8719 default: return Register();
8720}
8721}
8722
8723Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(MVT RetVT, Register Op0) {
8724 if (RetVT.SimpleTy != MVT::v2f64)
8725 return Register();
8726 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8727 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8728 }
8729 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8730 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
8731 }
8732 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8733 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
8734 }
8735 return Register();
8736}
8737
8738Register fastEmit_X86ISD_STRICT_VFPEXT_r(MVT VT, MVT RetVT, Register Op0) {
8739 switch (VT.SimpleTy) {
8740 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(RetVT, Op0);
8741 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(RetVT, Op0);
8742 default: return Register();
8743 }
8744}
8745
8746// FastEmit functions for X86ISD::STRICT_VFPROUND.
8747
8748Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
8749 if (RetVT.SimpleTy != MVT::v8f16)
8750 return Register();
8751 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8752 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
8753 }
8754 return Register();
8755}
8756
8757Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(MVT RetVT, Register Op0) {
8758 if (RetVT.SimpleTy != MVT::v8f16)
8759 return Register();
8760 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8761 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
8762 }
8763 return Register();
8764}
8765
8766Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(MVT RetVT, Register Op0) {
8767 if (RetVT.SimpleTy != MVT::v16f16)
8768 return Register();
8769 if ((Subtarget->hasFP16())) {
8770 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
8771 }
8772 return Register();
8773}
8774
8775Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Register Op0) {
8776 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8777 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8778 }
8779 return Register();
8780}
8781
8782Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Register Op0) {
8783 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8784 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8785 }
8786 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8787 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8788 }
8789 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8790 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8791 }
8792 return Register();
8793}
8794
8795Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
8796switch (RetVT.SimpleTy) {
8797 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
8798 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
8799 default: return Register();
8800}
8801}
8802
8803Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Register Op0) {
8804 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8805 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8806 }
8807 return Register();
8808}
8809
8810Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Register Op0) {
8811 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8812 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
8813 }
8814 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8815 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
8816 }
8817 return Register();
8818}
8819
8820Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(MVT RetVT, Register Op0) {
8821switch (RetVT.SimpleTy) {
8822 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
8823 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
8824 default: return Register();
8825}
8826}
8827
8828Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Register Op0) {
8829 if ((Subtarget->hasFP16())) {
8830 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
8831 }
8832 return Register();
8833}
8834
8835Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Register Op0) {
8836 if ((Subtarget->hasAVX512())) {
8837 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
8838 }
8839 return Register();
8840}
8841
8842Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(MVT RetVT, Register Op0) {
8843switch (RetVT.SimpleTy) {
8844 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
8845 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
8846 default: return Register();
8847}
8848}
8849
8850Register fastEmit_X86ISD_STRICT_VFPROUND_r(MVT VT, MVT RetVT, Register Op0) {
8851 switch (VT.SimpleTy) {
8852 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(RetVT, Op0);
8853 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(RetVT, Op0);
8854 case MVT::v16f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(RetVT, Op0);
8855 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(RetVT, Op0);
8856 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(RetVT, Op0);
8857 case MVT::v8f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(RetVT, Op0);
8858 default: return Register();
8859 }
8860}
8861
8862// FastEmit functions for X86ISD::VBROADCAST.
8863
8864Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Register Op0) {
8865 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8866 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ128rr, RC: &X86::VR128XRegClass, Op0);
8867 }
8868 return Register();
8869}
8870
8871Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Register Op0) {
8872 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8873 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ256rr, RC: &X86::VR256XRegClass, Op0);
8874 }
8875 return Register();
8876}
8877
8878Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Register Op0) {
8879 if ((Subtarget->hasAVX512())) {
8880 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZrr, RC: &X86::VR512RegClass, Op0);
8881 }
8882 return Register();
8883}
8884
8885Register fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, Register Op0) {
8886switch (RetVT.SimpleTy) {
8887 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0);
8888 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0);
8889 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0);
8890 default: return Register();
8891}
8892}
8893
8894Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Register Op0) {
8895 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8896 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ128rr, RC: &X86::VR128XRegClass, Op0);
8897 }
8898 return Register();
8899}
8900
8901Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Register Op0) {
8902 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8903 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ256rr, RC: &X86::VR256XRegClass, Op0);
8904 }
8905 return Register();
8906}
8907
8908Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Register Op0) {
8909 if ((Subtarget->hasAVX512())) {
8910 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZrr, RC: &X86::VR512RegClass, Op0);
8911 }
8912 return Register();
8913}
8914
8915Register fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, Register Op0) {
8916switch (RetVT.SimpleTy) {
8917 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0);
8918 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0);
8919 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0);
8920 default: return Register();
8921}
8922}
8923
8924Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Register Op0) {
8925 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8926 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ128rr, RC: &X86::VR128XRegClass, Op0);
8927 }
8928 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8929 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBrr, RC: &X86::VR128RegClass, Op0);
8930 }
8931 return Register();
8932}
8933
8934Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Register Op0) {
8935 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8936 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ256rr, RC: &X86::VR256XRegClass, Op0);
8937 }
8938 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8939 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBYrr, RC: &X86::VR256RegClass, Op0);
8940 }
8941 return Register();
8942}
8943
8944Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Register Op0) {
8945 if ((Subtarget->hasBWI())) {
8946 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZrr, RC: &X86::VR512RegClass, Op0);
8947 }
8948 return Register();
8949}
8950
8951Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
8952switch (RetVT.SimpleTy) {
8953 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0);
8954 case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0);
8955 case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0);
8956 default: return Register();
8957}
8958}
8959
8960Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Register Op0) {
8961 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8962 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
8963 }
8964 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8965 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
8966 }
8967 return Register();
8968}
8969
8970Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Register Op0) {
8971 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8972 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
8973 }
8974 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8975 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
8976 }
8977 return Register();
8978}
8979
8980Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Register Op0) {
8981 if ((Subtarget->hasBWI())) {
8982 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
8983 }
8984 return Register();
8985}
8986
8987Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
8988switch (RetVT.SimpleTy) {
8989 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0);
8990 case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0);
8991 case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0);
8992 default: return Register();
8993}
8994}
8995
8996Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Register Op0) {
8997 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8998 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ128rr, RC: &X86::VR128XRegClass, Op0);
8999 }
9000 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9001 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrr, RC: &X86::VR128RegClass, Op0);
9002 }
9003 return Register();
9004}
9005
9006Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Register Op0) {
9007 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9008 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ256rr, RC: &X86::VR256XRegClass, Op0);
9009 }
9010 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9011 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDYrr, RC: &X86::VR256RegClass, Op0);
9012 }
9013 return Register();
9014}
9015
9016Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Register Op0) {
9017 if ((Subtarget->hasAVX512())) {
9018 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZrr, RC: &X86::VR512RegClass, Op0);
9019 }
9020 return Register();
9021}
9022
9023Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
9024switch (RetVT.SimpleTy) {
9025 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0);
9026 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0);
9027 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0);
9028 default: return Register();
9029}
9030}
9031
9032Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Register Op0) {
9033 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9034 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ128rr, RC: &X86::VR128XRegClass, Op0);
9035 }
9036 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9037 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrr, RC: &X86::VR128RegClass, Op0);
9038 }
9039 return Register();
9040}
9041
9042Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Register Op0) {
9043 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9044 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ256rr, RC: &X86::VR256XRegClass, Op0);
9045 }
9046 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9047 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQYrr, RC: &X86::VR256RegClass, Op0);
9048 }
9049 return Register();
9050}
9051
9052Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Register Op0) {
9053 if ((Subtarget->hasAVX512())) {
9054 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZrr, RC: &X86::VR512RegClass, Op0);
9055 }
9056 return Register();
9057}
9058
9059Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
9060switch (RetVT.SimpleTy) {
9061 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0);
9062 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0);
9063 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0);
9064 default: return Register();
9065}
9066}
9067
9068Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Register Op0) {
9069 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9070 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
9071 }
9072 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9073 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
9074 }
9075 return Register();
9076}
9077
9078Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Register Op0) {
9079 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9080 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
9081 }
9082 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9083 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
9084 }
9085 return Register();
9086}
9087
9088Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Register Op0) {
9089 if ((Subtarget->hasBWI())) {
9090 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
9091 }
9092 return Register();
9093}
9094
9095Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
9096switch (RetVT.SimpleTy) {
9097 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Op0);
9098 case MVT::v16f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Op0);
9099 case MVT::v32f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Op0);
9100 default: return Register();
9101}
9102}
9103
9104Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Register Op0) {
9105 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9106 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
9107 }
9108 return Register();
9109}
9110
9111Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Register Op0) {
9112 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9113 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
9114 }
9115 return Register();
9116}
9117
9118Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Register Op0) {
9119 if ((Subtarget->hasBF16())) {
9120 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
9121 }
9122 return Register();
9123}
9124
9125Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
9126switch (RetVT.SimpleTy) {
9127 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Op0);
9128 case MVT::v16bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Op0);
9129 case MVT::v32bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Op0);
9130 default: return Register();
9131}
9132}
9133
9134Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Register Op0) {
9135 if ((Subtarget->hasVLX())) {
9136 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ128rr, RC: &X86::VR128XRegClass, Op0);
9137 }
9138 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9139 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSrr, RC: &X86::VR128RegClass, Op0);
9140 }
9141 return Register();
9142}
9143
9144Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Register Op0) {
9145 if ((Subtarget->hasVLX())) {
9146 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ256rr, RC: &X86::VR256XRegClass, Op0);
9147 }
9148 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9149 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSYrr, RC: &X86::VR256RegClass, Op0);
9150 }
9151 return Register();
9152}
9153
9154Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Register Op0) {
9155 if ((Subtarget->hasAVX512())) {
9156 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZrr, RC: &X86::VR512RegClass, Op0);
9157 }
9158 return Register();
9159}
9160
9161Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
9162switch (RetVT.SimpleTy) {
9163 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0);
9164 case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0);
9165 case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0);
9166 default: return Register();
9167}
9168}
9169
9170Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Register Op0) {
9171 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9172 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
9173 }
9174 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9175 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
9176 }
9177 return Register();
9178}
9179
9180Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Register Op0) {
9181 if ((Subtarget->hasVLX())) {
9182 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZ256rr, RC: &X86::VR256XRegClass, Op0);
9183 }
9184 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9185 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDYrr, RC: &X86::VR256RegClass, Op0);
9186 }
9187 return Register();
9188}
9189
9190Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Register Op0) {
9191 if ((Subtarget->hasAVX512())) {
9192 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZrr, RC: &X86::VR512RegClass, Op0);
9193 }
9194 return Register();
9195}
9196
9197Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
9198switch (RetVT.SimpleTy) {
9199 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0);
9200 case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0);
9201 case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0);
9202 default: return Register();
9203}
9204}
9205
9206Register fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, Register Op0) {
9207 switch (VT.SimpleTy) {
9208 case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0);
9209 case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0);
9210 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0);
9211 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0);
9212 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0);
9213 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0);
9214 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(RetVT, Op0);
9215 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(RetVT, Op0);
9216 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0);
9217 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0);
9218 default: return Register();
9219 }
9220}
9221
9222// FastEmit functions for X86ISD::VBROADCASTM.
9223
9224Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Register Op0) {
9225 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9226 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ128rr, RC: &X86::VR128XRegClass, Op0);
9227 }
9228 return Register();
9229}
9230
9231Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Register Op0) {
9232 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9233 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ256rr, RC: &X86::VR256XRegClass, Op0);
9234 }
9235 return Register();
9236}
9237
9238Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Register Op0) {
9239 if ((Subtarget->hasCDI())) {
9240 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZrr, RC: &X86::VR512RegClass, Op0);
9241 }
9242 return Register();
9243}
9244
9245Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, Register Op0) {
9246switch (RetVT.SimpleTy) {
9247 case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0);
9248 case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0);
9249 case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0);
9250 default: return Register();
9251}
9252}
9253
9254Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Register Op0) {
9255 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9256 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ128rr, RC: &X86::VR128XRegClass, Op0);
9257 }
9258 return Register();
9259}
9260
9261Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Register Op0) {
9262 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9263 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ256rr, RC: &X86::VR256XRegClass, Op0);
9264 }
9265 return Register();
9266}
9267
9268Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Register Op0) {
9269 if ((Subtarget->hasCDI())) {
9270 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZrr, RC: &X86::VR512RegClass, Op0);
9271 }
9272 return Register();
9273}
9274
9275Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, Register Op0) {
9276switch (RetVT.SimpleTy) {
9277 case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0);
9278 case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0);
9279 case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0);
9280 default: return Register();
9281}
9282}
9283
9284Register fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, Register Op0) {
9285 switch (VT.SimpleTy) {
9286 case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0);
9287 case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0);
9288 default: return Register();
9289 }
9290}
9291
9292// FastEmit functions for X86ISD::VCVTHF82PH.
9293
9294Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v8f16_r(Register Op0) {
9295 if ((Subtarget->hasAVX10_2())) {
9296 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZ128rr, RC: &X86::VR128XRegClass, Op0);
9297 }
9298 return Register();
9299}
9300
9301Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v16f16_r(Register Op0) {
9302 if ((Subtarget->hasAVX10_2())) {
9303 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZ256rr, RC: &X86::VR256XRegClass, Op0);
9304 }
9305 return Register();
9306}
9307
9308Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_r(MVT RetVT, Register Op0) {
9309switch (RetVT.SimpleTy) {
9310 case MVT::v8f16: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v8f16_r(Op0);
9311 case MVT::v16f16: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v16f16_r(Op0);
9312 default: return Register();
9313}
9314}
9315
9316Register fastEmit_X86ISD_VCVTHF82PH_MVT_v32i8_r(MVT RetVT, Register Op0) {
9317 if (RetVT.SimpleTy != MVT::v32f16)
9318 return Register();
9319 if ((Subtarget->hasAVX10_2_512())) {
9320 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZrr, RC: &X86::VR512RegClass, Op0);
9321 }
9322 return Register();
9323}
9324
9325Register fastEmit_X86ISD_VCVTHF82PH_r(MVT VT, MVT RetVT, Register Op0) {
9326 switch (VT.SimpleTy) {
9327 case MVT::v16i8: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_r(RetVT, Op0);
9328 case MVT::v32i8: return fastEmit_X86ISD_VCVTHF82PH_MVT_v32i8_r(RetVT, Op0);
9329 default: return Register();
9330 }
9331}
9332
9333// FastEmit functions for X86ISD::VCVTPH2BF8.
9334
9335Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v8f16_r(MVT RetVT, Register Op0) {
9336 if (RetVT.SimpleTy != MVT::v16i8)
9337 return Register();
9338 if ((Subtarget->hasAVX10_2())) {
9339 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9340 }
9341 return Register();
9342}
9343
9344Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v16f16_r(MVT RetVT, Register Op0) {
9345 if (RetVT.SimpleTy != MVT::v16i8)
9346 return Register();
9347 if ((Subtarget->hasAVX10_2())) {
9348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9349 }
9350 return Register();
9351}
9352
9353Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v32f16_r(MVT RetVT, Register Op0) {
9354 if (RetVT.SimpleTy != MVT::v32i8)
9355 return Register();
9356 if ((Subtarget->hasAVX10_2_512())) {
9357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Zrr, RC: &X86::VR256XRegClass, Op0);
9358 }
9359 return Register();
9360}
9361
9362Register fastEmit_X86ISD_VCVTPH2BF8_r(MVT VT, MVT RetVT, Register Op0) {
9363 switch (VT.SimpleTy) {
9364 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v8f16_r(RetVT, Op0);
9365 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v16f16_r(RetVT, Op0);
9366 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v32f16_r(RetVT, Op0);
9367 default: return Register();
9368 }
9369}
9370
9371// FastEmit functions for X86ISD::VCVTPH2BF8S.
9372
9373Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v8f16_r(MVT RetVT, Register Op0) {
9374 if (RetVT.SimpleTy != MVT::v16i8)
9375 return Register();
9376 if ((Subtarget->hasAVX10_2())) {
9377 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9378 }
9379 return Register();
9380}
9381
9382Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v16f16_r(MVT RetVT, Register Op0) {
9383 if (RetVT.SimpleTy != MVT::v16i8)
9384 return Register();
9385 if ((Subtarget->hasAVX10_2())) {
9386 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9387 }
9388 return Register();
9389}
9390
9391Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v32f16_r(MVT RetVT, Register Op0) {
9392 if (RetVT.SimpleTy != MVT::v32i8)
9393 return Register();
9394 if ((Subtarget->hasAVX10_2_512())) {
9395 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZrr, RC: &X86::VR256XRegClass, Op0);
9396 }
9397 return Register();
9398}
9399
9400Register fastEmit_X86ISD_VCVTPH2BF8S_r(MVT VT, MVT RetVT, Register Op0) {
9401 switch (VT.SimpleTy) {
9402 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v8f16_r(RetVT, Op0);
9403 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v16f16_r(RetVT, Op0);
9404 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v32f16_r(RetVT, Op0);
9405 default: return Register();
9406 }
9407}
9408
9409// FastEmit functions for X86ISD::VCVTPH2HF8.
9410
9411Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v8f16_r(MVT RetVT, Register Op0) {
9412 if (RetVT.SimpleTy != MVT::v16i8)
9413 return Register();
9414 if ((Subtarget->hasAVX10_2())) {
9415 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9416 }
9417 return Register();
9418}
9419
9420Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v16f16_r(MVT RetVT, Register Op0) {
9421 if (RetVT.SimpleTy != MVT::v16i8)
9422 return Register();
9423 if ((Subtarget->hasAVX10_2())) {
9424 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9425 }
9426 return Register();
9427}
9428
9429Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v32f16_r(MVT RetVT, Register Op0) {
9430 if (RetVT.SimpleTy != MVT::v32i8)
9431 return Register();
9432 if ((Subtarget->hasAVX10_2_512())) {
9433 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Zrr, RC: &X86::VR256XRegClass, Op0);
9434 }
9435 return Register();
9436}
9437
9438Register fastEmit_X86ISD_VCVTPH2HF8_r(MVT VT, MVT RetVT, Register Op0) {
9439 switch (VT.SimpleTy) {
9440 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v8f16_r(RetVT, Op0);
9441 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v16f16_r(RetVT, Op0);
9442 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v32f16_r(RetVT, Op0);
9443 default: return Register();
9444 }
9445}
9446
9447// FastEmit functions for X86ISD::VCVTPH2HF8S.
9448
9449Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v8f16_r(MVT RetVT, Register Op0) {
9450 if (RetVT.SimpleTy != MVT::v16i8)
9451 return Register();
9452 if ((Subtarget->hasAVX10_2())) {
9453 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9454 }
9455 return Register();
9456}
9457
9458Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v16f16_r(MVT RetVT, Register Op0) {
9459 if (RetVT.SimpleTy != MVT::v16i8)
9460 return Register();
9461 if ((Subtarget->hasAVX10_2())) {
9462 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9463 }
9464 return Register();
9465}
9466
9467Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v32f16_r(MVT RetVT, Register Op0) {
9468 if (RetVT.SimpleTy != MVT::v32i8)
9469 return Register();
9470 if ((Subtarget->hasAVX10_2_512())) {
9471 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZrr, RC: &X86::VR256XRegClass, Op0);
9472 }
9473 return Register();
9474}
9475
9476Register fastEmit_X86ISD_VCVTPH2HF8S_r(MVT VT, MVT RetVT, Register Op0) {
9477 switch (VT.SimpleTy) {
9478 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v8f16_r(RetVT, Op0);
9479 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v16f16_r(RetVT, Op0);
9480 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v32f16_r(RetVT, Op0);
9481 default: return Register();
9482 }
9483}
9484
9485// FastEmit functions for X86ISD::VFPEXT.
9486
9487Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Register Op0) {
9488 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9489 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
9490 }
9491 return Register();
9492}
9493
9494Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Register Op0) {
9495 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9496 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
9497 }
9498 return Register();
9499}
9500
9501Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Register Op0) {
9502 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9503 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
9504 }
9505 return Register();
9506}
9507
9508Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(MVT RetVT, Register Op0) {
9509switch (RetVT.SimpleTy) {
9510 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
9511 case MVT::v2f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
9512 case MVT::v4f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
9513 default: return Register();
9514}
9515}
9516
9517Register fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, Register Op0) {
9518 if (RetVT.SimpleTy != MVT::v2f64)
9519 return Register();
9520 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9521 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
9522 }
9523 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9524 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
9525 }
9526 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9527 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
9528 }
9529 return Register();
9530}
9531
9532Register fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, Register Op0) {
9533 switch (VT.SimpleTy) {
9534 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(RetVT, Op0);
9535 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0);
9536 default: return Register();
9537 }
9538}
9539
9540// FastEmit functions for X86ISD::VFPEXT_SAE.
9541
9542Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
9543 if (RetVT.SimpleTy != MVT::v8f64)
9544 return Register();
9545 if ((Subtarget->hasFP16())) {
9546 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrrb, RC: &X86::VR512RegClass, Op0);
9547 }
9548 return Register();
9549}
9550
9551Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
9552 if (RetVT.SimpleTy != MVT::v16f32)
9553 return Register();
9554 if ((Subtarget->hasFP16())) {
9555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrrb, RC: &X86::VR512RegClass, Op0);
9556 }
9557 return Register();
9558}
9559
9560Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
9561 if (RetVT.SimpleTy != MVT::v8f64)
9562 return Register();
9563 if ((Subtarget->hasAVX512())) {
9564 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrrb, RC: &X86::VR512RegClass, Op0);
9565 }
9566 return Register();
9567}
9568
9569Register fastEmit_X86ISD_VFPEXT_SAE_r(MVT VT, MVT RetVT, Register Op0) {
9570 switch (VT.SimpleTy) {
9571 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(RetVT, Op0);
9572 case MVT::v16f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(RetVT, Op0);
9573 case MVT::v8f32: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(RetVT, Op0);
9574 default: return Register();
9575 }
9576}
9577
9578// FastEmit functions for X86ISD::VFPROUND.
9579
9580Register fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
9581 if (RetVT.SimpleTy != MVT::v8f16)
9582 return Register();
9583 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9584 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
9585 }
9586 return Register();
9587}
9588
9589Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Register Op0) {
9590 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9591 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
9592 }
9593 return Register();
9594}
9595
9596Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Register Op0) {
9597 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9598 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
9599 }
9600 if ((Subtarget->hasAVXNECONVERT())) {
9601 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Yrr, RC: &X86::VR128RegClass, Op0);
9602 }
9603 return Register();
9604}
9605
9606Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(MVT RetVT, Register Op0) {
9607switch (RetVT.SimpleTy) {
9608 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Op0);
9609 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Op0);
9610 default: return Register();
9611}
9612}
9613
9614Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Register Op0) {
9615 if ((Subtarget->hasFP16())) {
9616 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
9617 }
9618 return Register();
9619}
9620
9621Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Register Op0) {
9622 if ((Subtarget->hasBF16())) {
9623 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
9624 }
9625 return Register();
9626}
9627
9628Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(MVT RetVT, Register Op0) {
9629switch (RetVT.SimpleTy) {
9630 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Op0);
9631 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Op0);
9632 default: return Register();
9633}
9634}
9635
9636Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Register Op0) {
9637 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9638 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
9639 }
9640 return Register();
9641}
9642
9643Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Register Op0) {
9644 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9645 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
9646 }
9647 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9648 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
9649 }
9650 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9651 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
9652 }
9653 return Register();
9654}
9655
9656Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
9657switch (RetVT.SimpleTy) {
9658 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
9659 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
9660 default: return Register();
9661}
9662}
9663
9664Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Register Op0) {
9665 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9666 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
9667 }
9668 return Register();
9669}
9670
9671Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Register Op0) {
9672 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9673 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
9674 }
9675 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9676 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
9677 }
9678 return Register();
9679}
9680
9681Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(MVT RetVT, Register Op0) {
9682switch (RetVT.SimpleTy) {
9683 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
9684 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
9685 default: return Register();
9686}
9687}
9688
9689Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Register Op0) {
9690 if ((Subtarget->hasFP16())) {
9691 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
9692 }
9693 return Register();
9694}
9695
9696Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Register Op0) {
9697 if ((Subtarget->hasAVX512())) {
9698 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
9699 }
9700 return Register();
9701}
9702
9703Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(MVT RetVT, Register Op0) {
9704switch (RetVT.SimpleTy) {
9705 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
9706 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
9707 default: return Register();
9708}
9709}
9710
9711Register fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, Register Op0) {
9712 switch (VT.SimpleTy) {
9713 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(RetVT, Op0);
9714 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(RetVT, Op0);
9715 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(RetVT, Op0);
9716 case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0);
9717 case MVT::v4f64: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(RetVT, Op0);
9718 case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(RetVT, Op0);
9719 default: return Register();
9720 }
9721}
9722
9723// FastEmit functions for X86ISD::VTRUNC.
9724
9725Register fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, Register Op0) {
9726 if (RetVT.SimpleTy != MVT::v16i8)
9727 return Register();
9728 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9729 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ128rr, RC: &X86::VR128XRegClass, Op0);
9730 }
9731 return Register();
9732}
9733
9734Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Register Op0) {
9735 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9736 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ128rr, RC: &X86::VR128XRegClass, Op0);
9737 }
9738 return Register();
9739}
9740
9741Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Register Op0) {
9742 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9743 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ128rr, RC: &X86::VR128XRegClass, Op0);
9744 }
9745 return Register();
9746}
9747
9748Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, Register Op0) {
9749switch (RetVT.SimpleTy) {
9750 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0);
9751 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0);
9752 default: return Register();
9753}
9754}
9755
9756Register fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, Register Op0) {
9757 if (RetVT.SimpleTy != MVT::v16i8)
9758 return Register();
9759 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9760 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ256rr, RC: &X86::VR128XRegClass, Op0);
9761 }
9762 return Register();
9763}
9764
9765Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Register Op0) {
9766 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9767 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ128rr, RC: &X86::VR128XRegClass, Op0);
9768 }
9769 return Register();
9770}
9771
9772Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Register Op0) {
9773 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9774 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ128rr, RC: &X86::VR128XRegClass, Op0);
9775 }
9776 return Register();
9777}
9778
9779Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Register Op0) {
9780 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9781 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ128rr, RC: &X86::VR128XRegClass, Op0);
9782 }
9783 return Register();
9784}
9785
9786Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, Register Op0) {
9787switch (RetVT.SimpleTy) {
9788 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0);
9789 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0);
9790 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0);
9791 default: return Register();
9792}
9793}
9794
9795Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Register Op0) {
9796 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9797 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ256rr, RC: &X86::VR128XRegClass, Op0);
9798 }
9799 return Register();
9800}
9801
9802Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Register Op0) {
9803 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9804 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ256rr, RC: &X86::VR128XRegClass, Op0);
9805 }
9806 return Register();
9807}
9808
9809Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, Register Op0) {
9810switch (RetVT.SimpleTy) {
9811 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0);
9812 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0);
9813 default: return Register();
9814}
9815}
9816
9817Register fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, Register Op0) {
9818 if (RetVT.SimpleTy != MVT::v16i8)
9819 return Register();
9820 if ((Subtarget->hasAVX512())) {
9821 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZrr, RC: &X86::VR128XRegClass, Op0);
9822 }
9823 return Register();
9824}
9825
9826Register fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
9827 switch (VT.SimpleTy) {
9828 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0);
9829 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0);
9830 case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0);
9831 case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0);
9832 case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0);
9833 case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0);
9834 default: return Register();
9835 }
9836}
9837
9838// FastEmit functions for X86ISD::VTRUNCS.
9839
9840Register fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, Register Op0) {
9841 if (RetVT.SimpleTy != MVT::v16i8)
9842 return Register();
9843 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9844 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
9845 }
9846 return Register();
9847}
9848
9849Register fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, Register Op0) {
9850 if (RetVT.SimpleTy != MVT::v16i8)
9851 return Register();
9852 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9853 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
9854 }
9855 return Register();
9856}
9857
9858Register fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, Register Op0) {
9859 if (RetVT.SimpleTy != MVT::v32i8)
9860 return Register();
9861 if ((Subtarget->hasBWI())) {
9862 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZrr, RC: &X86::VR256XRegClass, Op0);
9863 }
9864 return Register();
9865}
9866
9867Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Register Op0) {
9868 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9869 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
9870 }
9871 return Register();
9872}
9873
9874Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Register Op0) {
9875 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9876 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
9877 }
9878 return Register();
9879}
9880
9881Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, Register Op0) {
9882switch (RetVT.SimpleTy) {
9883 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0);
9884 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0);
9885 default: return Register();
9886}
9887}
9888
9889Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Register Op0) {
9890 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9891 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
9892 }
9893 return Register();
9894}
9895
9896Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Register Op0) {
9897 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9898 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
9899 }
9900 return Register();
9901}
9902
9903Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, Register Op0) {
9904switch (RetVT.SimpleTy) {
9905 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0);
9906 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0);
9907 default: return Register();
9908}
9909}
9910
9911Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Register Op0) {
9912 if ((Subtarget->hasAVX512())) {
9913 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZrr, RC: &X86::VR128XRegClass, Op0);
9914 }
9915 return Register();
9916}
9917
9918Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Register Op0) {
9919 if ((Subtarget->hasAVX512())) {
9920 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZrr, RC: &X86::VR256XRegClass, Op0);
9921 }
9922 return Register();
9923}
9924
9925Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, Register Op0) {
9926switch (RetVT.SimpleTy) {
9927 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0);
9928 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0);
9929 default: return Register();
9930}
9931}
9932
9933Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Register Op0) {
9934 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9935 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
9936 }
9937 return Register();
9938}
9939
9940Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Register Op0) {
9941 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9942 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
9943 }
9944 return Register();
9945}
9946
9947Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Register Op0) {
9948 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9949 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
9950 }
9951 return Register();
9952}
9953
9954Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, Register Op0) {
9955switch (RetVT.SimpleTy) {
9956 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0);
9957 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0);
9958 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0);
9959 default: return Register();
9960}
9961}
9962
9963Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Register Op0) {
9964 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9965 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
9966 }
9967 return Register();
9968}
9969
9970Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Register Op0) {
9971 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9972 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
9973 }
9974 return Register();
9975}
9976
9977Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Register Op0) {
9978 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9979 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
9980 }
9981 return Register();
9982}
9983
9984Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, Register Op0) {
9985switch (RetVT.SimpleTy) {
9986 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0);
9987 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0);
9988 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0);
9989 default: return Register();
9990}
9991}
9992
9993Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Register Op0) {
9994 if ((Subtarget->hasAVX512())) {
9995 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZrr, RC: &X86::VR128XRegClass, Op0);
9996 }
9997 return Register();
9998}
9999
10000Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Register Op0) {
10001 if ((Subtarget->hasAVX512())) {
10002 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZrr, RC: &X86::VR128XRegClass, Op0);
10003 }
10004 return Register();
10005}
10006
10007Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Register Op0) {
10008 if ((Subtarget->hasAVX512())) {
10009 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZrr, RC: &X86::VR256XRegClass, Op0);
10010 }
10011 return Register();
10012}
10013
10014Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, Register Op0) {
10015switch (RetVT.SimpleTy) {
10016 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0);
10017 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0);
10018 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0);
10019 default: return Register();
10020}
10021}
10022
10023Register fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, Register Op0) {
10024 switch (VT.SimpleTy) {
10025 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0);
10026 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0);
10027 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0);
10028 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0);
10029 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0);
10030 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0);
10031 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0);
10032 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0);
10033 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0);
10034 default: return Register();
10035 }
10036}
10037
10038// FastEmit functions for X86ISD::VTRUNCUS.
10039
10040Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, Register Op0) {
10041 if (RetVT.SimpleTy != MVT::v16i8)
10042 return Register();
10043 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10044 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
10045 }
10046 return Register();
10047}
10048
10049Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, Register Op0) {
10050 if (RetVT.SimpleTy != MVT::v16i8)
10051 return Register();
10052 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10053 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
10054 }
10055 return Register();
10056}
10057
10058Register fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, Register Op0) {
10059 if (RetVT.SimpleTy != MVT::v32i8)
10060 return Register();
10061 if ((Subtarget->hasBWI())) {
10062 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZrr, RC: &X86::VR256XRegClass, Op0);
10063 }
10064 return Register();
10065}
10066
10067Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Register Op0) {
10068 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10069 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
10070 }
10071 return Register();
10072}
10073
10074Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Register Op0) {
10075 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10076 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
10077 }
10078 return Register();
10079}
10080
10081Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, Register Op0) {
10082switch (RetVT.SimpleTy) {
10083 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0);
10084 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0);
10085 default: return Register();
10086}
10087}
10088
10089Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Register Op0) {
10090 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10091 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
10092 }
10093 return Register();
10094}
10095
10096Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Register Op0) {
10097 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10098 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
10099 }
10100 return Register();
10101}
10102
10103Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, Register Op0) {
10104switch (RetVT.SimpleTy) {
10105 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0);
10106 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0);
10107 default: return Register();
10108}
10109}
10110
10111Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Register Op0) {
10112 if ((Subtarget->hasAVX512())) {
10113 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZrr, RC: &X86::VR128XRegClass, Op0);
10114 }
10115 return Register();
10116}
10117
10118Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Register Op0) {
10119 if ((Subtarget->hasAVX512())) {
10120 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZrr, RC: &X86::VR256XRegClass, Op0);
10121 }
10122 return Register();
10123}
10124
10125Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, Register Op0) {
10126switch (RetVT.SimpleTy) {
10127 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0);
10128 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0);
10129 default: return Register();
10130}
10131}
10132
10133Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Register Op0) {
10134 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10135 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
10136 }
10137 return Register();
10138}
10139
10140Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Register Op0) {
10141 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10142 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
10143 }
10144 return Register();
10145}
10146
10147Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Register Op0) {
10148 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10149 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
10150 }
10151 return Register();
10152}
10153
10154Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, Register Op0) {
10155switch (RetVT.SimpleTy) {
10156 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0);
10157 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0);
10158 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0);
10159 default: return Register();
10160}
10161}
10162
10163Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Register Op0) {
10164 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10165 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
10166 }
10167 return Register();
10168}
10169
10170Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Register Op0) {
10171 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10172 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
10173 }
10174 return Register();
10175}
10176
10177Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Register Op0) {
10178 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10179 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
10180 }
10181 return Register();
10182}
10183
10184Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, Register Op0) {
10185switch (RetVT.SimpleTy) {
10186 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0);
10187 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0);
10188 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0);
10189 default: return Register();
10190}
10191}
10192
10193Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Register Op0) {
10194 if ((Subtarget->hasAVX512())) {
10195 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZrr, RC: &X86::VR128XRegClass, Op0);
10196 }
10197 return Register();
10198}
10199
10200Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Register Op0) {
10201 if ((Subtarget->hasAVX512())) {
10202 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZrr, RC: &X86::VR128XRegClass, Op0);
10203 }
10204 return Register();
10205}
10206
10207Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Register Op0) {
10208 if ((Subtarget->hasAVX512())) {
10209 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZrr, RC: &X86::VR256XRegClass, Op0);
10210 }
10211 return Register();
10212}
10213
10214Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, Register Op0) {
10215switch (RetVT.SimpleTy) {
10216 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0);
10217 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0);
10218 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0);
10219 default: return Register();
10220}
10221}
10222
10223Register fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, Register Op0) {
10224 switch (VT.SimpleTy) {
10225 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0);
10226 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0);
10227 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0);
10228 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0);
10229 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0);
10230 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0);
10231 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0);
10232 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0);
10233 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0);
10234 default: return Register();
10235 }
10236}
10237
10238// FastEmit functions for X86ISD::VZEXT_MOVL.
10239
10240Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v8i16_r(MVT RetVT, Register Op0) {
10241 if (RetVT.SimpleTy != MVT::v8i16)
10242 return Register();
10243 if ((Subtarget->hasAVX10_2())) {
10244 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPWILo2PWIZrr, RC: &X86::VR128XRegClass, Op0);
10245 }
10246 return Register();
10247}
10248
10249Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v4i32_r(MVT RetVT, Register Op0) {
10250 if (RetVT.SimpleTy != MVT::v4i32)
10251 return Register();
10252 if ((Subtarget->hasAVX10_2())) {
10253 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPDILo2PDIZrr, RC: &X86::VR128XRegClass, Op0);
10254 }
10255 return Register();
10256}
10257
10258Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, Register Op0) {
10259 if (RetVT.SimpleTy != MVT::v2i64)
10260 return Register();
10261 if ((Subtarget->hasAVX512())) {
10262 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
10263 }
10264 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10265 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10266 }
10267 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10268 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10269 }
10270 return Register();
10271}
10272
10273Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, Register Op0) {
10274 if (RetVT.SimpleTy != MVT::v2f64)
10275 return Register();
10276 if ((Subtarget->hasAVX512())) {
10277 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
10278 }
10279 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10280 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10281 }
10282 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10283 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10284 }
10285 return Register();
10286}
10287
10288Register fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, Register Op0) {
10289 switch (VT.SimpleTy) {
10290 case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v8i16_r(RetVT, Op0);
10291 case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v4i32_r(RetVT, Op0);
10292 case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0);
10293 case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0);
10294 default: return Register();
10295 }
10296}
10297
10298// Top-level FastEmit function.
10299
10300Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
10301 switch (Opcode) {
10302 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
10303 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
10304 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
10305 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
10306 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
10307 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
10308 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
10309 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
10310 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
10311 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
10312 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
10313 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
10314 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
10315 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
10316 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
10317 case ISD::LLRINT: return fastEmit_ISD_LLRINT_r(VT, RetVT, Op0);
10318 case ISD::LRINT: return fastEmit_ISD_LRINT_r(VT, RetVT, Op0);
10319 case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0);
10320 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
10321 case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
10322 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
10323 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
10324 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
10325 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
10326 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
10327 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
10328 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
10329 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
10330 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
10331 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
10332 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
10333 case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
10334 case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0);
10335 case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0);
10336 case X86ISD::CVTNEPS2BF16: return fastEmit_X86ISD_CVTNEPS2BF16_r(VT, RetVT, Op0);
10337 case X86ISD::CVTP2IBS: return fastEmit_X86ISD_CVTP2IBS_r(VT, RetVT, Op0);
10338 case X86ISD::CVTP2IUBS: return fastEmit_X86ISD_CVTP2IUBS_r(VT, RetVT, Op0);
10339 case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0);
10340 case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0);
10341 case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0);
10342 case X86ISD::CVTPH2PS_SAE: return fastEmit_X86ISD_CVTPH2PS_SAE_r(VT, RetVT, Op0);
10343 case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0);
10344 case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0);
10345 case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0);
10346 case X86ISD::CVTTP2IBS: return fastEmit_X86ISD_CVTTP2IBS_r(VT, RetVT, Op0);
10347 case X86ISD::CVTTP2IBS_SAE: return fastEmit_X86ISD_CVTTP2IBS_SAE_r(VT, RetVT, Op0);
10348 case X86ISD::CVTTP2IUBS: return fastEmit_X86ISD_CVTTP2IUBS_r(VT, RetVT, Op0);
10349 case X86ISD::CVTTP2IUBS_SAE: return fastEmit_X86ISD_CVTTP2IUBS_SAE_r(VT, RetVT, Op0);
10350 case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0);
10351 case X86ISD::CVTTP2SIS: return fastEmit_X86ISD_CVTTP2SIS_r(VT, RetVT, Op0);
10352 case X86ISD::CVTTP2SIS_SAE: return fastEmit_X86ISD_CVTTP2SIS_SAE_r(VT, RetVT, Op0);
10353 case X86ISD::CVTTP2SI_SAE: return fastEmit_X86ISD_CVTTP2SI_SAE_r(VT, RetVT, Op0);
10354 case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0);
10355 case X86ISD::CVTTP2UIS: return fastEmit_X86ISD_CVTTP2UIS_r(VT, RetVT, Op0);
10356 case X86ISD::CVTTP2UIS_SAE: return fastEmit_X86ISD_CVTTP2UIS_SAE_r(VT, RetVT, Op0);
10357 case X86ISD::CVTTP2UI_SAE: return fastEmit_X86ISD_CVTTP2UI_SAE_r(VT, RetVT, Op0);
10358 case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0);
10359 case X86ISD::CVTTS2SIS: return fastEmit_X86ISD_CVTTS2SIS_r(VT, RetVT, Op0);
10360 case X86ISD::CVTTS2SIS_SAE: return fastEmit_X86ISD_CVTTS2SIS_SAE_r(VT, RetVT, Op0);
10361 case X86ISD::CVTTS2SI_SAE: return fastEmit_X86ISD_CVTTS2SI_SAE_r(VT, RetVT, Op0);
10362 case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0);
10363 case X86ISD::CVTTS2UIS: return fastEmit_X86ISD_CVTTS2UIS_r(VT, RetVT, Op0);
10364 case X86ISD::CVTTS2UIS_SAE: return fastEmit_X86ISD_CVTTS2UIS_SAE_r(VT, RetVT, Op0);
10365 case X86ISD::CVTTS2UI_SAE: return fastEmit_X86ISD_CVTTS2UI_SAE_r(VT, RetVT, Op0);
10366 case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0);
10367 case X86ISD::DYN_ALLOCA: return fastEmit_X86ISD_DYN_ALLOCA_r(VT, RetVT, Op0);
10368 case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0);
10369 case X86ISD::FGETEXP: return fastEmit_X86ISD_FGETEXP_r(VT, RetVT, Op0);
10370 case X86ISD::FGETEXP_SAE: return fastEmit_X86ISD_FGETEXP_SAE_r(VT, RetVT, Op0);
10371 case X86ISD::FP_TO_SINT_SAT: return fastEmit_X86ISD_FP_TO_SINT_SAT_r(VT, RetVT, Op0);
10372 case X86ISD::FP_TO_UINT_SAT: return fastEmit_X86ISD_FP_TO_UINT_SAT_r(VT, RetVT, Op0);
10373 case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0);
10374 case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0);
10375 case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0);
10376 case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0);
10377 case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0);
10378 case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0);
10379 case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0);
10380 case X86ISD::MOVQ2DQ: return fastEmit_X86ISD_MOVQ2DQ_r(VT, RetVT, Op0);
10381 case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0);
10382 case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0);
10383 case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0);
10384 case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0);
10385 case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0);
10386 case X86ISD::PROBED_ALLOCA: return fastEmit_X86ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
10387 case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0);
10388 case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0);
10389 case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0);
10390 case X86ISD::STRICT_CVTPH2PS: return fastEmit_X86ISD_STRICT_CVTPH2PS_r(VT, RetVT, Op0);
10391 case X86ISD::STRICT_CVTSI2P: return fastEmit_X86ISD_STRICT_CVTSI2P_r(VT, RetVT, Op0);
10392 case X86ISD::STRICT_CVTTP2SI: return fastEmit_X86ISD_STRICT_CVTTP2SI_r(VT, RetVT, Op0);
10393 case X86ISD::STRICT_CVTTP2UI: return fastEmit_X86ISD_STRICT_CVTTP2UI_r(VT, RetVT, Op0);
10394 case X86ISD::STRICT_CVTUI2P: return fastEmit_X86ISD_STRICT_CVTUI2P_r(VT, RetVT, Op0);
10395 case X86ISD::STRICT_VFPEXT: return fastEmit_X86ISD_STRICT_VFPEXT_r(VT, RetVT, Op0);
10396 case X86ISD::STRICT_VFPROUND: return fastEmit_X86ISD_STRICT_VFPROUND_r(VT, RetVT, Op0);
10397 case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0);
10398 case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0);
10399 case X86ISD::VCVTHF82PH: return fastEmit_X86ISD_VCVTHF82PH_r(VT, RetVT, Op0);
10400 case X86ISD::VCVTPH2BF8: return fastEmit_X86ISD_VCVTPH2BF8_r(VT, RetVT, Op0);
10401 case X86ISD::VCVTPH2BF8S: return fastEmit_X86ISD_VCVTPH2BF8S_r(VT, RetVT, Op0);
10402 case X86ISD::VCVTPH2HF8: return fastEmit_X86ISD_VCVTPH2HF8_r(VT, RetVT, Op0);
10403 case X86ISD::VCVTPH2HF8S: return fastEmit_X86ISD_VCVTPH2HF8S_r(VT, RetVT, Op0);
10404 case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0);
10405 case X86ISD::VFPEXT_SAE: return fastEmit_X86ISD_VFPEXT_SAE_r(VT, RetVT, Op0);
10406 case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0);
10407 case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0);
10408 case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0);
10409 case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0);
10410 case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0);
10411 default: return Register();
10412 }
10413}
10414
10415// FastEmit functions for ISD::ADD.
10416
10417Register fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
10418 if (RetVT.SimpleTy != MVT::i8)
10419 return Register();
10420 if ((Subtarget->hasNDD())) {
10421 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10422 }
10423 if ((!Subtarget->hasNDD())) {
10424 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr, RC: &X86::GR8RegClass, Op0, Op1);
10425 }
10426 return Register();
10427}
10428
10429Register fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
10430 if (RetVT.SimpleTy != MVT::i16)
10431 return Register();
10432 if ((Subtarget->hasNDD())) {
10433 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10434 }
10435 if ((!Subtarget->hasNDD())) {
10436 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr, RC: &X86::GR16RegClass, Op0, Op1);
10437 }
10438 return Register();
10439}
10440
10441Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10442 if (RetVT.SimpleTy != MVT::i32)
10443 return Register();
10444 if ((Subtarget->hasNDD())) {
10445 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10446 }
10447 if ((!Subtarget->hasNDD())) {
10448 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr, RC: &X86::GR32RegClass, Op0, Op1);
10449 }
10450 return Register();
10451}
10452
10453Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10454 if (RetVT.SimpleTy != MVT::i64)
10455 return Register();
10456 if ((Subtarget->hasNDD())) {
10457 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10458 }
10459 if ((!Subtarget->hasNDD())) {
10460 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr, RC: &X86::GR64RegClass, Op0, Op1);
10461 }
10462 return Register();
10463}
10464
10465Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10466 if (RetVT.SimpleTy != MVT::v16i8)
10467 return Register();
10468 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10469 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10470 }
10471 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10472 return fastEmitInst_rr(MachineInstOpcode: X86::PADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
10473 }
10474 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10475 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
10476 }
10477 return Register();
10478}
10479
10480Register fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10481 if (RetVT.SimpleTy != MVT::v32i8)
10482 return Register();
10483 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10484 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10485 }
10486 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10487 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10488 }
10489 return Register();
10490}
10491
10492Register fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10493 if (RetVT.SimpleTy != MVT::v64i8)
10494 return Register();
10495 if ((Subtarget->hasBWI())) {
10496 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10497 }
10498 return Register();
10499}
10500
10501Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10502 if (RetVT.SimpleTy != MVT::v8i16)
10503 return Register();
10504 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10505 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10506 }
10507 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10508 return fastEmitInst_rr(MachineInstOpcode: X86::PADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
10509 }
10510 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10511 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
10512 }
10513 return Register();
10514}
10515
10516Register fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10517 if (RetVT.SimpleTy != MVT::v16i16)
10518 return Register();
10519 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10520 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10521 }
10522 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10523 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10524 }
10525 return Register();
10526}
10527
10528Register fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10529 if (RetVT.SimpleTy != MVT::v32i16)
10530 return Register();
10531 if ((Subtarget->hasBWI())) {
10532 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10533 }
10534 return Register();
10535}
10536
10537Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10538 if (RetVT.SimpleTy != MVT::v4i32)
10539 return Register();
10540 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10541 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10542 }
10543 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10544 return fastEmitInst_rr(MachineInstOpcode: X86::PADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
10545 }
10546 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10547 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
10548 }
10549 return Register();
10550}
10551
10552Register fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
10553 if (RetVT.SimpleTy != MVT::v8i32)
10554 return Register();
10555 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10556 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10557 }
10558 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10559 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10560 }
10561 return Register();
10562}
10563
10564Register fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
10565 if (RetVT.SimpleTy != MVT::v16i32)
10566 return Register();
10567 if ((Subtarget->hasAVX512())) {
10568 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10569 }
10570 return Register();
10571}
10572
10573Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10574 if (RetVT.SimpleTy != MVT::v2i64)
10575 return Register();
10576 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10577 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10578 }
10579 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10580 return fastEmitInst_rr(MachineInstOpcode: X86::PADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
10581 }
10582 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10583 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
10584 }
10585 return Register();
10586}
10587
10588Register fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
10589 if (RetVT.SimpleTy != MVT::v4i64)
10590 return Register();
10591 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10592 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10593 }
10594 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10595 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
10596 }
10597 return Register();
10598}
10599
10600Register fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
10601 if (RetVT.SimpleTy != MVT::v8i64)
10602 return Register();
10603 if ((Subtarget->hasAVX512())) {
10604 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10605 }
10606 return Register();
10607}
10608
10609Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10610 switch (VT.SimpleTy) {
10611 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op1);
10612 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op1);
10613 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10614 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10615 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10616 case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op1);
10617 case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op1);
10618 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10619 case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op1);
10620 case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op1);
10621 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10622 case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op1);
10623 case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op1);
10624 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10625 case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op1);
10626 case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op1);
10627 default: return Register();
10628 }
10629}
10630
10631// FastEmit functions for ISD::AND.
10632
10633Register fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
10634 if (RetVT.SimpleTy != MVT::i8)
10635 return Register();
10636 if ((Subtarget->hasNDD())) {
10637 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10638 }
10639 if ((!Subtarget->hasNDD())) {
10640 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr, RC: &X86::GR8RegClass, Op0, Op1);
10641 }
10642 return Register();
10643}
10644
10645Register fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
10646 if (RetVT.SimpleTy != MVT::i16)
10647 return Register();
10648 if ((Subtarget->hasNDD())) {
10649 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10650 }
10651 if ((!Subtarget->hasNDD())) {
10652 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr, RC: &X86::GR16RegClass, Op0, Op1);
10653 }
10654 return Register();
10655}
10656
10657Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10658 if (RetVT.SimpleTy != MVT::i32)
10659 return Register();
10660 if ((Subtarget->hasNDD())) {
10661 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10662 }
10663 if ((!Subtarget->hasNDD())) {
10664 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr, RC: &X86::GR32RegClass, Op0, Op1);
10665 }
10666 return Register();
10667}
10668
10669Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10670 if (RetVT.SimpleTy != MVT::i64)
10671 return Register();
10672 if ((Subtarget->hasNDD())) {
10673 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10674 }
10675 if ((!Subtarget->hasNDD())) {
10676 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr, RC: &X86::GR64RegClass, Op0, Op1);
10677 }
10678 return Register();
10679}
10680
10681Register fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10682 if (RetVT.SimpleTy != MVT::v8i1)
10683 return Register();
10684 if ((Subtarget->hasDQI())) {
10685 return fastEmitInst_rr(MachineInstOpcode: X86::KANDBkk, RC: &X86::VK8RegClass, Op0, Op1);
10686 }
10687 return Register();
10688}
10689
10690Register fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
10691 if (RetVT.SimpleTy != MVT::v16i1)
10692 return Register();
10693 if ((Subtarget->hasAVX512())) {
10694 return fastEmitInst_rr(MachineInstOpcode: X86::KANDWkk, RC: &X86::VK16RegClass, Op0, Op1);
10695 }
10696 return Register();
10697}
10698
10699Register fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
10700 if (RetVT.SimpleTy != MVT::v32i1)
10701 return Register();
10702 if ((Subtarget->hasBWI())) {
10703 return fastEmitInst_rr(MachineInstOpcode: X86::KANDDkk, RC: &X86::VK32RegClass, Op0, Op1);
10704 }
10705 return Register();
10706}
10707
10708Register fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
10709 if (RetVT.SimpleTy != MVT::v64i1)
10710 return Register();
10711 if ((Subtarget->hasBWI())) {
10712 return fastEmitInst_rr(MachineInstOpcode: X86::KANDQkk, RC: &X86::VK64RegClass, Op0, Op1);
10713 }
10714 return Register();
10715}
10716
10717Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10718 if (RetVT.SimpleTy != MVT::v16i8)
10719 return Register();
10720 if ((Subtarget->hasVLX())) {
10721 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10722 }
10723 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10724 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10725 }
10726 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10727 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10728 }
10729 return Register();
10730}
10731
10732Register fastEmit_ISD_AND_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10733 if (RetVT.SimpleTy != MVT::v32i8)
10734 return Register();
10735 if ((Subtarget->hasVLX())) {
10736 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10737 }
10738 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10739 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10740 }
10741 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10742 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10743 }
10744 return Register();
10745}
10746
10747Register fastEmit_ISD_AND_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10748 if (RetVT.SimpleTy != MVT::v64i8)
10749 return Register();
10750 if ((Subtarget->hasAVX512())) {
10751 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10752 }
10753 return Register();
10754}
10755
10756Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10757 if (RetVT.SimpleTy != MVT::v8i16)
10758 return Register();
10759 if ((Subtarget->hasVLX())) {
10760 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10761 }
10762 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10763 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10764 }
10765 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10766 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10767 }
10768 return Register();
10769}
10770
10771Register fastEmit_ISD_AND_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10772 if (RetVT.SimpleTy != MVT::v16i16)
10773 return Register();
10774 if ((Subtarget->hasVLX())) {
10775 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10776 }
10777 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10778 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10779 }
10780 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10781 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10782 }
10783 return Register();
10784}
10785
10786Register fastEmit_ISD_AND_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10787 if (RetVT.SimpleTy != MVT::v32i16)
10788 return Register();
10789 if ((Subtarget->hasAVX512())) {
10790 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10791 }
10792 return Register();
10793}
10794
10795Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10796 if (RetVT.SimpleTy != MVT::v4i32)
10797 return Register();
10798 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10799 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10800 }
10801 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10802 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10803 }
10804 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10805 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10806 }
10807 return Register();
10808}
10809
10810Register fastEmit_ISD_AND_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
10811 if (RetVT.SimpleTy != MVT::v8i32)
10812 return Register();
10813 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10814 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10815 }
10816 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10817 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10818 }
10819 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10820 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10821 }
10822 return Register();
10823}
10824
10825Register fastEmit_ISD_AND_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
10826 if (RetVT.SimpleTy != MVT::v16i32)
10827 return Register();
10828 if ((Subtarget->hasAVX512())) {
10829 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10830 }
10831 return Register();
10832}
10833
10834Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10835 if (RetVT.SimpleTy != MVT::v2i64)
10836 return Register();
10837 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10838 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10839 }
10840 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10841 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10842 }
10843 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10844 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10845 }
10846 return Register();
10847}
10848
10849Register fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
10850 if (RetVT.SimpleTy != MVT::v4i64)
10851 return Register();
10852 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10853 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10854 }
10855 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10856 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10857 }
10858 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10859 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10860 }
10861 return Register();
10862}
10863
10864Register fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
10865 if (RetVT.SimpleTy != MVT::v8i64)
10866 return Register();
10867 if ((Subtarget->hasAVX512())) {
10868 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10869 }
10870 return Register();
10871}
10872
10873Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10874 switch (VT.SimpleTy) {
10875 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op1);
10876 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op1);
10877 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10878 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10879 case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op1);
10880 case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op1);
10881 case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op1);
10882 case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op1);
10883 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10884 case MVT::v32i8: return fastEmit_ISD_AND_MVT_v32i8_rr(RetVT, Op0, Op1);
10885 case MVT::v64i8: return fastEmit_ISD_AND_MVT_v64i8_rr(RetVT, Op0, Op1);
10886 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10887 case MVT::v16i16: return fastEmit_ISD_AND_MVT_v16i16_rr(RetVT, Op0, Op1);
10888 case MVT::v32i16: return fastEmit_ISD_AND_MVT_v32i16_rr(RetVT, Op0, Op1);
10889 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10890 case MVT::v8i32: return fastEmit_ISD_AND_MVT_v8i32_rr(RetVT, Op0, Op1);
10891 case MVT::v16i32: return fastEmit_ISD_AND_MVT_v16i32_rr(RetVT, Op0, Op1);
10892 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10893 case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op1);
10894 case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op1);
10895 default: return Register();
10896 }
10897}
10898
10899// FastEmit functions for ISD::AVGCEILU.
10900
10901Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10902 if (RetVT.SimpleTy != MVT::v16i8)
10903 return Register();
10904 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10905 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10906 }
10907 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10908 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
10909 }
10910 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10911 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
10912 }
10913 return Register();
10914}
10915
10916Register fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10917 if (RetVT.SimpleTy != MVT::v32i8)
10918 return Register();
10919 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10920 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10921 }
10922 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10923 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10924 }
10925 return Register();
10926}
10927
10928Register fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10929 if (RetVT.SimpleTy != MVT::v64i8)
10930 return Register();
10931 if ((Subtarget->hasBWI())) {
10932 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10933 }
10934 return Register();
10935}
10936
10937Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10938 if (RetVT.SimpleTy != MVT::v8i16)
10939 return Register();
10940 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10941 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10942 }
10943 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10944 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
10945 }
10946 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10947 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
10948 }
10949 return Register();
10950}
10951
10952Register fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10953 if (RetVT.SimpleTy != MVT::v16i16)
10954 return Register();
10955 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10956 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10957 }
10958 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10959 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10960 }
10961 return Register();
10962}
10963
10964Register fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10965 if (RetVT.SimpleTy != MVT::v32i16)
10966 return Register();
10967 if ((Subtarget->hasBWI())) {
10968 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10969 }
10970 return Register();
10971}
10972
10973Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10974 switch (VT.SimpleTy) {
10975 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10976 case MVT::v32i8: return fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(RetVT, Op0, Op1);
10977 case MVT::v64i8: return fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(RetVT, Op0, Op1);
10978 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10979 case MVT::v16i16: return fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(RetVT, Op0, Op1);
10980 case MVT::v32i16: return fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(RetVT, Op0, Op1);
10981 default: return Register();
10982 }
10983}
10984
10985// FastEmit functions for ISD::FADD.
10986
10987Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10988 if (RetVT.SimpleTy != MVT::f16)
10989 return Register();
10990 if ((Subtarget->hasFP16())) {
10991 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
10992 }
10993 return Register();
10994}
10995
10996Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10997 if (RetVT.SimpleTy != MVT::f32)
10998 return Register();
10999 if ((Subtarget->hasAVX512())) {
11000 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11001 }
11002 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11003 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11004 }
11005 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11006 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11007 }
11008 if ((!Subtarget->hasSSE1())) {
11009 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11010 }
11011 return Register();
11012}
11013
11014Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11015 if (RetVT.SimpleTy != MVT::f64)
11016 return Register();
11017 if ((Subtarget->hasAVX512())) {
11018 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11019 }
11020 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11021 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11022 }
11023 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11024 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11025 }
11026 if ((!Subtarget->hasSSE2())) {
11027 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11028 }
11029 return Register();
11030}
11031
11032Register fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11033 if (RetVT.SimpleTy != MVT::f80)
11034 return Register();
11035 if ((Subtarget->hasX87())) {
11036 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11037 }
11038 return Register();
11039}
11040
11041Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11042 if (RetVT.SimpleTy != MVT::v8f16)
11043 return Register();
11044 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11045 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11046 }
11047 return Register();
11048}
11049
11050Register fastEmit_ISD_FADD_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11051 if (RetVT.SimpleTy != MVT::v16f16)
11052 return Register();
11053 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11054 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11055 }
11056 return Register();
11057}
11058
11059Register fastEmit_ISD_FADD_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11060 if (RetVT.SimpleTy != MVT::v32f16)
11061 return Register();
11062 if ((Subtarget->hasFP16())) {
11063 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11064 }
11065 return Register();
11066}
11067
11068Register fastEmit_ISD_FADD_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11069 if (RetVT.SimpleTy != MVT::v8bf16)
11070 return Register();
11071 if ((Subtarget->hasAVX10_2())) {
11072 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11073 }
11074 return Register();
11075}
11076
11077Register fastEmit_ISD_FADD_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11078 if (RetVT.SimpleTy != MVT::v16bf16)
11079 return Register();
11080 if ((Subtarget->hasAVX10_2())) {
11081 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11082 }
11083 return Register();
11084}
11085
11086Register fastEmit_ISD_FADD_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11087 if (RetVT.SimpleTy != MVT::v32bf16)
11088 return Register();
11089 if ((Subtarget->hasAVX10_2_512())) {
11090 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11091 }
11092 return Register();
11093}
11094
11095Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11096 if (RetVT.SimpleTy != MVT::v4f32)
11097 return Register();
11098 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11099 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11100 }
11101 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11102 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11103 }
11104 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11105 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11106 }
11107 return Register();
11108}
11109
11110Register fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11111 if (RetVT.SimpleTy != MVT::v8f32)
11112 return Register();
11113 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11114 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11115 }
11116 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11117 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11118 }
11119 return Register();
11120}
11121
11122Register fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11123 if (RetVT.SimpleTy != MVT::v16f32)
11124 return Register();
11125 if ((Subtarget->hasAVX512())) {
11126 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11127 }
11128 return Register();
11129}
11130
11131Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11132 if (RetVT.SimpleTy != MVT::v2f64)
11133 return Register();
11134 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11135 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11136 }
11137 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11138 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11139 }
11140 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11141 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11142 }
11143 return Register();
11144}
11145
11146Register fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11147 if (RetVT.SimpleTy != MVT::v4f64)
11148 return Register();
11149 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11150 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11151 }
11152 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11153 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11154 }
11155 return Register();
11156}
11157
11158Register fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11159 if (RetVT.SimpleTy != MVT::v8f64)
11160 return Register();
11161 if ((Subtarget->hasAVX512())) {
11162 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11163 }
11164 return Register();
11165}
11166
11167Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11168 switch (VT.SimpleTy) {
11169 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11170 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11171 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11172 case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op1);
11173 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11174 case MVT::v16f16: return fastEmit_ISD_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
11175 case MVT::v32f16: return fastEmit_ISD_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
11176 case MVT::v8bf16: return fastEmit_ISD_FADD_MVT_v8bf16_rr(RetVT, Op0, Op1);
11177 case MVT::v16bf16: return fastEmit_ISD_FADD_MVT_v16bf16_rr(RetVT, Op0, Op1);
11178 case MVT::v32bf16: return fastEmit_ISD_FADD_MVT_v32bf16_rr(RetVT, Op0, Op1);
11179 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11180 case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
11181 case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
11182 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11183 case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
11184 case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
11185 default: return Register();
11186 }
11187}
11188
11189// FastEmit functions for ISD::FDIV.
11190
11191Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11192 if (RetVT.SimpleTy != MVT::f16)
11193 return Register();
11194 if ((Subtarget->hasFP16())) {
11195 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11196 }
11197 return Register();
11198}
11199
11200Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11201 if (RetVT.SimpleTy != MVT::f32)
11202 return Register();
11203 if ((Subtarget->hasAVX512())) {
11204 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11205 }
11206 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11207 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11208 }
11209 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11210 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11211 }
11212 if ((!Subtarget->hasSSE1())) {
11213 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11214 }
11215 return Register();
11216}
11217
11218Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11219 if (RetVT.SimpleTy != MVT::f64)
11220 return Register();
11221 if ((Subtarget->hasAVX512())) {
11222 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11223 }
11224 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11225 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11226 }
11227 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11228 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11229 }
11230 if ((!Subtarget->hasSSE2())) {
11231 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11232 }
11233 return Register();
11234}
11235
11236Register fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11237 if (RetVT.SimpleTy != MVT::f80)
11238 return Register();
11239 if ((Subtarget->hasX87())) {
11240 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11241 }
11242 return Register();
11243}
11244
11245Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11246 if (RetVT.SimpleTy != MVT::v8f16)
11247 return Register();
11248 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11249 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11250 }
11251 return Register();
11252}
11253
11254Register fastEmit_ISD_FDIV_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11255 if (RetVT.SimpleTy != MVT::v16f16)
11256 return Register();
11257 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11258 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11259 }
11260 return Register();
11261}
11262
11263Register fastEmit_ISD_FDIV_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11264 if (RetVT.SimpleTy != MVT::v32f16)
11265 return Register();
11266 if ((Subtarget->hasFP16())) {
11267 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11268 }
11269 return Register();
11270}
11271
11272Register fastEmit_ISD_FDIV_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11273 if (RetVT.SimpleTy != MVT::v8bf16)
11274 return Register();
11275 if ((Subtarget->hasAVX10_2())) {
11276 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11277 }
11278 return Register();
11279}
11280
11281Register fastEmit_ISD_FDIV_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11282 if (RetVT.SimpleTy != MVT::v16bf16)
11283 return Register();
11284 if ((Subtarget->hasAVX10_2())) {
11285 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11286 }
11287 return Register();
11288}
11289
11290Register fastEmit_ISD_FDIV_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11291 if (RetVT.SimpleTy != MVT::v32bf16)
11292 return Register();
11293 if ((Subtarget->hasAVX10_2_512())) {
11294 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11295 }
11296 return Register();
11297}
11298
11299Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11300 if (RetVT.SimpleTy != MVT::v4f32)
11301 return Register();
11302 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11303 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11304 }
11305 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11306 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11307 }
11308 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11309 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11310 }
11311 return Register();
11312}
11313
11314Register fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11315 if (RetVT.SimpleTy != MVT::v8f32)
11316 return Register();
11317 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11318 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11319 }
11320 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11321 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11322 }
11323 return Register();
11324}
11325
11326Register fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11327 if (RetVT.SimpleTy != MVT::v16f32)
11328 return Register();
11329 if ((Subtarget->hasAVX512())) {
11330 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11331 }
11332 return Register();
11333}
11334
11335Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11336 if (RetVT.SimpleTy != MVT::v2f64)
11337 return Register();
11338 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11339 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11340 }
11341 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11342 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11343 }
11344 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11345 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11346 }
11347 return Register();
11348}
11349
11350Register fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11351 if (RetVT.SimpleTy != MVT::v4f64)
11352 return Register();
11353 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11354 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11355 }
11356 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11357 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11358 }
11359 return Register();
11360}
11361
11362Register fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11363 if (RetVT.SimpleTy != MVT::v8f64)
11364 return Register();
11365 if ((Subtarget->hasAVX512())) {
11366 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11367 }
11368 return Register();
11369}
11370
11371Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11372 switch (VT.SimpleTy) {
11373 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11374 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11375 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11376 case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
11377 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11378 case MVT::v16f16: return fastEmit_ISD_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
11379 case MVT::v32f16: return fastEmit_ISD_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
11380 case MVT::v8bf16: return fastEmit_ISD_FDIV_MVT_v8bf16_rr(RetVT, Op0, Op1);
11381 case MVT::v16bf16: return fastEmit_ISD_FDIV_MVT_v16bf16_rr(RetVT, Op0, Op1);
11382 case MVT::v32bf16: return fastEmit_ISD_FDIV_MVT_v32bf16_rr(RetVT, Op0, Op1);
11383 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11384 case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
11385 case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
11386 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11387 case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
11388 case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
11389 default: return Register();
11390 }
11391}
11392
11393// FastEmit functions for ISD::FMUL.
11394
11395Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11396 if (RetVT.SimpleTy != MVT::f16)
11397 return Register();
11398 if ((Subtarget->hasFP16())) {
11399 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11400 }
11401 return Register();
11402}
11403
11404Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11405 if (RetVT.SimpleTy != MVT::f32)
11406 return Register();
11407 if ((Subtarget->hasAVX512())) {
11408 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11409 }
11410 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11411 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11412 }
11413 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11414 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11415 }
11416 if ((!Subtarget->hasSSE1())) {
11417 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11418 }
11419 return Register();
11420}
11421
11422Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11423 if (RetVT.SimpleTy != MVT::f64)
11424 return Register();
11425 if ((Subtarget->hasAVX512())) {
11426 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11427 }
11428 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11429 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11430 }
11431 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11432 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11433 }
11434 if ((!Subtarget->hasSSE2())) {
11435 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11436 }
11437 return Register();
11438}
11439
11440Register fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11441 if (RetVT.SimpleTy != MVT::f80)
11442 return Register();
11443 if ((Subtarget->hasX87())) {
11444 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11445 }
11446 return Register();
11447}
11448
11449Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11450 if (RetVT.SimpleTy != MVT::v8f16)
11451 return Register();
11452 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11453 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11454 }
11455 return Register();
11456}
11457
11458Register fastEmit_ISD_FMUL_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11459 if (RetVT.SimpleTy != MVT::v16f16)
11460 return Register();
11461 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11462 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11463 }
11464 return Register();
11465}
11466
11467Register fastEmit_ISD_FMUL_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11468 if (RetVT.SimpleTy != MVT::v32f16)
11469 return Register();
11470 if ((Subtarget->hasFP16())) {
11471 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11472 }
11473 return Register();
11474}
11475
11476Register fastEmit_ISD_FMUL_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11477 if (RetVT.SimpleTy != MVT::v8bf16)
11478 return Register();
11479 if ((Subtarget->hasAVX10_2())) {
11480 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11481 }
11482 return Register();
11483}
11484
11485Register fastEmit_ISD_FMUL_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11486 if (RetVT.SimpleTy != MVT::v16bf16)
11487 return Register();
11488 if ((Subtarget->hasAVX10_2())) {
11489 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11490 }
11491 return Register();
11492}
11493
11494Register fastEmit_ISD_FMUL_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11495 if (RetVT.SimpleTy != MVT::v32bf16)
11496 return Register();
11497 if ((Subtarget->hasAVX10_2_512())) {
11498 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11499 }
11500 return Register();
11501}
11502
11503Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11504 if (RetVT.SimpleTy != MVT::v4f32)
11505 return Register();
11506 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11507 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11508 }
11509 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11510 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11511 }
11512 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11513 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11514 }
11515 return Register();
11516}
11517
11518Register fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11519 if (RetVT.SimpleTy != MVT::v8f32)
11520 return Register();
11521 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11522 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11523 }
11524 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11525 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11526 }
11527 return Register();
11528}
11529
11530Register fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11531 if (RetVT.SimpleTy != MVT::v16f32)
11532 return Register();
11533 if ((Subtarget->hasAVX512())) {
11534 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11535 }
11536 return Register();
11537}
11538
11539Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11540 if (RetVT.SimpleTy != MVT::v2f64)
11541 return Register();
11542 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11543 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11544 }
11545 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11546 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11547 }
11548 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11549 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11550 }
11551 return Register();
11552}
11553
11554Register fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11555 if (RetVT.SimpleTy != MVT::v4f64)
11556 return Register();
11557 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11558 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11559 }
11560 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11561 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11562 }
11563 return Register();
11564}
11565
11566Register fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11567 if (RetVT.SimpleTy != MVT::v8f64)
11568 return Register();
11569 if ((Subtarget->hasAVX512())) {
11570 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11571 }
11572 return Register();
11573}
11574
11575Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11576 switch (VT.SimpleTy) {
11577 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11578 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11579 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11580 case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
11581 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11582 case MVT::v16f16: return fastEmit_ISD_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
11583 case MVT::v32f16: return fastEmit_ISD_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
11584 case MVT::v8bf16: return fastEmit_ISD_FMUL_MVT_v8bf16_rr(RetVT, Op0, Op1);
11585 case MVT::v16bf16: return fastEmit_ISD_FMUL_MVT_v16bf16_rr(RetVT, Op0, Op1);
11586 case MVT::v32bf16: return fastEmit_ISD_FMUL_MVT_v32bf16_rr(RetVT, Op0, Op1);
11587 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11588 case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
11589 case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
11590 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11591 case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
11592 case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
11593 default: return Register();
11594 }
11595}
11596
11597// FastEmit functions for ISD::FSUB.
11598
11599Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11600 if (RetVT.SimpleTy != MVT::f16)
11601 return Register();
11602 if ((Subtarget->hasFP16())) {
11603 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11604 }
11605 return Register();
11606}
11607
11608Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11609 if (RetVT.SimpleTy != MVT::f32)
11610 return Register();
11611 if ((Subtarget->hasAVX512())) {
11612 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11613 }
11614 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11615 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11616 }
11617 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11618 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11619 }
11620 if ((!Subtarget->hasSSE1())) {
11621 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11622 }
11623 return Register();
11624}
11625
11626Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11627 if (RetVT.SimpleTy != MVT::f64)
11628 return Register();
11629 if ((Subtarget->hasAVX512())) {
11630 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11631 }
11632 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11633 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11634 }
11635 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11636 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11637 }
11638 if ((!Subtarget->hasSSE2())) {
11639 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11640 }
11641 return Register();
11642}
11643
11644Register fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11645 if (RetVT.SimpleTy != MVT::f80)
11646 return Register();
11647 if ((Subtarget->hasX87())) {
11648 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11649 }
11650 return Register();
11651}
11652
11653Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11654 if (RetVT.SimpleTy != MVT::v8f16)
11655 return Register();
11656 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11657 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11658 }
11659 return Register();
11660}
11661
11662Register fastEmit_ISD_FSUB_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11663 if (RetVT.SimpleTy != MVT::v16f16)
11664 return Register();
11665 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11666 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11667 }
11668 return Register();
11669}
11670
11671Register fastEmit_ISD_FSUB_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11672 if (RetVT.SimpleTy != MVT::v32f16)
11673 return Register();
11674 if ((Subtarget->hasFP16())) {
11675 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11676 }
11677 return Register();
11678}
11679
11680Register fastEmit_ISD_FSUB_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11681 if (RetVT.SimpleTy != MVT::v8bf16)
11682 return Register();
11683 if ((Subtarget->hasAVX10_2())) {
11684 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11685 }
11686 return Register();
11687}
11688
11689Register fastEmit_ISD_FSUB_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11690 if (RetVT.SimpleTy != MVT::v16bf16)
11691 return Register();
11692 if ((Subtarget->hasAVX10_2())) {
11693 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11694 }
11695 return Register();
11696}
11697
11698Register fastEmit_ISD_FSUB_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11699 if (RetVT.SimpleTy != MVT::v32bf16)
11700 return Register();
11701 if ((Subtarget->hasAVX10_2_512())) {
11702 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11703 }
11704 return Register();
11705}
11706
11707Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11708 if (RetVT.SimpleTy != MVT::v4f32)
11709 return Register();
11710 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11711 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11712 }
11713 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11714 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11715 }
11716 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11717 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11718 }
11719 return Register();
11720}
11721
11722Register fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11723 if (RetVT.SimpleTy != MVT::v8f32)
11724 return Register();
11725 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11726 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11727 }
11728 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11729 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11730 }
11731 return Register();
11732}
11733
11734Register fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11735 if (RetVT.SimpleTy != MVT::v16f32)
11736 return Register();
11737 if ((Subtarget->hasAVX512())) {
11738 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11739 }
11740 return Register();
11741}
11742
11743Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11744 if (RetVT.SimpleTy != MVT::v2f64)
11745 return Register();
11746 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11747 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11748 }
11749 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11750 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11751 }
11752 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11753 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11754 }
11755 return Register();
11756}
11757
11758Register fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11759 if (RetVT.SimpleTy != MVT::v4f64)
11760 return Register();
11761 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11762 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11763 }
11764 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11765 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11766 }
11767 return Register();
11768}
11769
11770Register fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11771 if (RetVT.SimpleTy != MVT::v8f64)
11772 return Register();
11773 if ((Subtarget->hasAVX512())) {
11774 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11775 }
11776 return Register();
11777}
11778
11779Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11780 switch (VT.SimpleTy) {
11781 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11782 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11783 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11784 case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
11785 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11786 case MVT::v16f16: return fastEmit_ISD_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
11787 case MVT::v32f16: return fastEmit_ISD_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
11788 case MVT::v8bf16: return fastEmit_ISD_FSUB_MVT_v8bf16_rr(RetVT, Op0, Op1);
11789 case MVT::v16bf16: return fastEmit_ISD_FSUB_MVT_v16bf16_rr(RetVT, Op0, Op1);
11790 case MVT::v32bf16: return fastEmit_ISD_FSUB_MVT_v32bf16_rr(RetVT, Op0, Op1);
11791 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11792 case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
11793 case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
11794 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11795 case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
11796 case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
11797 default: return Register();
11798 }
11799}
11800
11801// FastEmit functions for ISD::MUL.
11802
11803Register fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
11804 if (RetVT.SimpleTy != MVT::i8)
11805 return Register();
11806 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::AL).addReg(RegNo: Op0);
11807 return fastEmitInst_r(MachineInstOpcode: X86::MUL8r, RC: &X86::GR8RegClass, Op0: Op1);
11808}
11809
11810Register fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
11811 if (RetVT.SimpleTy != MVT::i16)
11812 return Register();
11813 if ((Subtarget->hasNDD())) {
11814 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
11815 }
11816 if ((!Subtarget->hasNDD())) {
11817 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr, RC: &X86::GR16RegClass, Op0, Op1);
11818 }
11819 return Register();
11820}
11821
11822Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11823 if (RetVT.SimpleTy != MVT::i32)
11824 return Register();
11825 if ((Subtarget->hasNDD())) {
11826 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
11827 }
11828 if ((!Subtarget->hasNDD())) {
11829 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr, RC: &X86::GR32RegClass, Op0, Op1);
11830 }
11831 return Register();
11832}
11833
11834Register fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11835 if (RetVT.SimpleTy != MVT::i64)
11836 return Register();
11837 if ((Subtarget->hasNDD())) {
11838 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
11839 }
11840 if ((!Subtarget->hasNDD())) {
11841 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr, RC: &X86::GR64RegClass, Op0, Op1);
11842 }
11843 return Register();
11844}
11845
11846Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11847 if (RetVT.SimpleTy != MVT::v8i16)
11848 return Register();
11849 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11850 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11851 }
11852 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11853 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
11854 }
11855 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11856 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
11857 }
11858 return Register();
11859}
11860
11861Register fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
11862 if (RetVT.SimpleTy != MVT::v16i16)
11863 return Register();
11864 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11865 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11866 }
11867 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11868 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWYrr, RC: &X86::VR256RegClass, Op0, Op1);
11869 }
11870 return Register();
11871}
11872
11873Register fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
11874 if (RetVT.SimpleTy != MVT::v32i16)
11875 return Register();
11876 if ((Subtarget->hasBWI())) {
11877 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11878 }
11879 return Register();
11880}
11881
11882Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11883 if (RetVT.SimpleTy != MVT::v4i32)
11884 return Register();
11885 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11886 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11887 }
11888 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
11889 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
11890 }
11891 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11892 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
11893 }
11894 return Register();
11895}
11896
11897Register fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
11898 if (RetVT.SimpleTy != MVT::v8i32)
11899 return Register();
11900 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11901 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11902 }
11903 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11904 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11905 }
11906 return Register();
11907}
11908
11909Register fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
11910 if (RetVT.SimpleTy != MVT::v16i32)
11911 return Register();
11912 if ((Subtarget->hasAVX512())) {
11913 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11914 }
11915 return Register();
11916}
11917
11918Register fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11919 if (RetVT.SimpleTy != MVT::v2i64)
11920 return Register();
11921 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
11922 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11923 }
11924 return Register();
11925}
11926
11927Register fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
11928 if (RetVT.SimpleTy != MVT::v4i64)
11929 return Register();
11930 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
11931 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11932 }
11933 return Register();
11934}
11935
11936Register fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
11937 if (RetVT.SimpleTy != MVT::v8i64)
11938 return Register();
11939 if ((Subtarget->hasDQI())) {
11940 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZrr, RC: &X86::VR512RegClass, Op0, Op1);
11941 }
11942 return Register();
11943}
11944
11945Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11946 switch (VT.SimpleTy) {
11947 case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op1);
11948 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op1);
11949 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
11950 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op1);
11951 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
11952 case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op1);
11953 case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op1);
11954 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
11955 case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op1);
11956 case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op1);
11957 case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op1);
11958 case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op1);
11959 case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op1);
11960 default: return Register();
11961 }
11962}
11963
11964// FastEmit functions for ISD::MULHS.
11965
11966Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11967 if (RetVT.SimpleTy != MVT::v8i16)
11968 return Register();
11969 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11970 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11971 }
11972 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11973 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
11974 }
11975 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11976 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
11977 }
11978 return Register();
11979}
11980
11981Register fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
11982 if (RetVT.SimpleTy != MVT::v16i16)
11983 return Register();
11984 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11985 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11986 }
11987 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11988 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWYrr, RC: &X86::VR256RegClass, Op0, Op1);
11989 }
11990 return Register();
11991}
11992
11993Register fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
11994 if (RetVT.SimpleTy != MVT::v32i16)
11995 return Register();
11996 if ((Subtarget->hasBWI())) {
11997 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11998 }
11999 return Register();
12000}
12001
12002Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12003 switch (VT.SimpleTy) {
12004 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
12005 case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op1);
12006 case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op1);
12007 default: return Register();
12008 }
12009}
12010
12011// FastEmit functions for ISD::MULHU.
12012
12013Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12014 if (RetVT.SimpleTy != MVT::v8i16)
12015 return Register();
12016 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12017 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12018 }
12019 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12020 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12021 }
12022 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12023 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12024 }
12025 return Register();
12026}
12027
12028Register fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12029 if (RetVT.SimpleTy != MVT::v16i16)
12030 return Register();
12031 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12032 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12033 }
12034 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12035 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12036 }
12037 return Register();
12038}
12039
12040Register fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12041 if (RetVT.SimpleTy != MVT::v32i16)
12042 return Register();
12043 if ((Subtarget->hasBWI())) {
12044 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12045 }
12046 return Register();
12047}
12048
12049Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12050 switch (VT.SimpleTy) {
12051 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
12052 case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op1);
12053 case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op1);
12054 default: return Register();
12055 }
12056}
12057
12058// FastEmit functions for ISD::OR.
12059
12060Register fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12061 if (RetVT.SimpleTy != MVT::i8)
12062 return Register();
12063 if ((Subtarget->hasNDD())) {
12064 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
12065 }
12066 if ((!Subtarget->hasNDD())) {
12067 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr, RC: &X86::GR8RegClass, Op0, Op1);
12068 }
12069 return Register();
12070}
12071
12072Register fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
12073 if (RetVT.SimpleTy != MVT::i16)
12074 return Register();
12075 if ((Subtarget->hasNDD())) {
12076 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
12077 }
12078 if ((!Subtarget->hasNDD())) {
12079 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr, RC: &X86::GR16RegClass, Op0, Op1);
12080 }
12081 return Register();
12082}
12083
12084Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12085 if (RetVT.SimpleTy != MVT::i32)
12086 return Register();
12087 if ((Subtarget->hasNDD())) {
12088 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
12089 }
12090 if ((!Subtarget->hasNDD())) {
12091 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr, RC: &X86::GR32RegClass, Op0, Op1);
12092 }
12093 return Register();
12094}
12095
12096Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12097 if (RetVT.SimpleTy != MVT::i64)
12098 return Register();
12099 if ((Subtarget->hasNDD())) {
12100 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
12101 }
12102 if ((!Subtarget->hasNDD())) {
12103 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr, RC: &X86::GR64RegClass, Op0, Op1);
12104 }
12105 return Register();
12106}
12107
12108Register fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
12109 if (RetVT.SimpleTy != MVT::v8i1)
12110 return Register();
12111 if ((Subtarget->hasDQI())) {
12112 return fastEmitInst_rr(MachineInstOpcode: X86::KORBkk, RC: &X86::VK8RegClass, Op0, Op1);
12113 }
12114 return Register();
12115}
12116
12117Register fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
12118 if (RetVT.SimpleTy != MVT::v16i1)
12119 return Register();
12120 if ((Subtarget->hasAVX512())) {
12121 return fastEmitInst_rr(MachineInstOpcode: X86::KORWkk, RC: &X86::VK16RegClass, Op0, Op1);
12122 }
12123 return Register();
12124}
12125
12126Register fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
12127 if (RetVT.SimpleTy != MVT::v32i1)
12128 return Register();
12129 if ((Subtarget->hasBWI())) {
12130 return fastEmitInst_rr(MachineInstOpcode: X86::KORDkk, RC: &X86::VK32RegClass, Op0, Op1);
12131 }
12132 return Register();
12133}
12134
12135Register fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
12136 if (RetVT.SimpleTy != MVT::v64i1)
12137 return Register();
12138 if ((Subtarget->hasBWI())) {
12139 return fastEmitInst_rr(MachineInstOpcode: X86::KORQkk, RC: &X86::VK64RegClass, Op0, Op1);
12140 }
12141 return Register();
12142}
12143
12144Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12145 if (RetVT.SimpleTy != MVT::v16i8)
12146 return Register();
12147 if ((Subtarget->hasVLX())) {
12148 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12149 }
12150 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12151 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12152 }
12153 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12154 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12155 }
12156 return Register();
12157}
12158
12159Register fastEmit_ISD_OR_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12160 if (RetVT.SimpleTy != MVT::v32i8)
12161 return Register();
12162 if ((Subtarget->hasVLX())) {
12163 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12164 }
12165 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12166 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12167 }
12168 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12169 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12170 }
12171 return Register();
12172}
12173
12174Register fastEmit_ISD_OR_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12175 if (RetVT.SimpleTy != MVT::v64i8)
12176 return Register();
12177 if ((Subtarget->hasAVX512())) {
12178 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12179 }
12180 return Register();
12181}
12182
12183Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12184 if (RetVT.SimpleTy != MVT::v8i16)
12185 return Register();
12186 if ((Subtarget->hasVLX())) {
12187 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12188 }
12189 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12190 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12191 }
12192 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12193 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12194 }
12195 return Register();
12196}
12197
12198Register fastEmit_ISD_OR_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12199 if (RetVT.SimpleTy != MVT::v16i16)
12200 return Register();
12201 if ((Subtarget->hasVLX())) {
12202 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12203 }
12204 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12205 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12206 }
12207 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12208 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12209 }
12210 return Register();
12211}
12212
12213Register fastEmit_ISD_OR_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12214 if (RetVT.SimpleTy != MVT::v32i16)
12215 return Register();
12216 if ((Subtarget->hasAVX512())) {
12217 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12218 }
12219 return Register();
12220}
12221
12222Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12223 if (RetVT.SimpleTy != MVT::v4i32)
12224 return Register();
12225 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12226 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12227 }
12228 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12229 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12230 }
12231 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12232 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12233 }
12234 return Register();
12235}
12236
12237Register fastEmit_ISD_OR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12238 if (RetVT.SimpleTy != MVT::v8i32)
12239 return Register();
12240 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12241 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12242 }
12243 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12244 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12245 }
12246 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12247 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12248 }
12249 return Register();
12250}
12251
12252Register fastEmit_ISD_OR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12253 if (RetVT.SimpleTy != MVT::v16i32)
12254 return Register();
12255 if ((Subtarget->hasAVX512())) {
12256 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12257 }
12258 return Register();
12259}
12260
12261Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12262 if (RetVT.SimpleTy != MVT::v2i64)
12263 return Register();
12264 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12265 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12266 }
12267 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12268 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12269 }
12270 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12271 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12272 }
12273 return Register();
12274}
12275
12276Register fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12277 if (RetVT.SimpleTy != MVT::v4i64)
12278 return Register();
12279 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12280 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12281 }
12282 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12283 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12284 }
12285 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12286 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12287 }
12288 return Register();
12289}
12290
12291Register fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12292 if (RetVT.SimpleTy != MVT::v8i64)
12293 return Register();
12294 if ((Subtarget->hasAVX512())) {
12295 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12296 }
12297 return Register();
12298}
12299
12300Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12301 switch (VT.SimpleTy) {
12302 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op1);
12303 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op1);
12304 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12305 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12306 case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op1);
12307 case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op1);
12308 case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op1);
12309 case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op1);
12310 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12311 case MVT::v32i8: return fastEmit_ISD_OR_MVT_v32i8_rr(RetVT, Op0, Op1);
12312 case MVT::v64i8: return fastEmit_ISD_OR_MVT_v64i8_rr(RetVT, Op0, Op1);
12313 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12314 case MVT::v16i16: return fastEmit_ISD_OR_MVT_v16i16_rr(RetVT, Op0, Op1);
12315 case MVT::v32i16: return fastEmit_ISD_OR_MVT_v32i16_rr(RetVT, Op0, Op1);
12316 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12317 case MVT::v8i32: return fastEmit_ISD_OR_MVT_v8i32_rr(RetVT, Op0, Op1);
12318 case MVT::v16i32: return fastEmit_ISD_OR_MVT_v16i32_rr(RetVT, Op0, Op1);
12319 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12320 case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op1);
12321 case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op1);
12322 default: return Register();
12323 }
12324}
12325
12326// FastEmit functions for ISD::ROTL.
12327
12328Register fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12329 if (RetVT.SimpleTy != MVT::i8)
12330 return Register();
12331 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12332 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12333 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL_ND, RC: &X86::GR8RegClass, Op0);
12334 }
12335 if ((!Subtarget->hasNDD())) {
12336 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12337 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL, RC: &X86::GR8RegClass, Op0);
12338 }
12339 return Register();
12340}
12341
12342Register fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12343 if (RetVT.SimpleTy != MVT::v16i8)
12344 return Register();
12345 if ((Subtarget->hasXOP())) {
12346 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTBrr, RC: &X86::VR128RegClass, Op0, Op1);
12347 }
12348 return Register();
12349}
12350
12351Register fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12352 if (RetVT.SimpleTy != MVT::v8i16)
12353 return Register();
12354 if ((Subtarget->hasXOP())) {
12355 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTWrr, RC: &X86::VR128RegClass, Op0, Op1);
12356 }
12357 return Register();
12358}
12359
12360Register fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12361 if (RetVT.SimpleTy != MVT::v4i32)
12362 return Register();
12363 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12364 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12365 }
12366 if ((Subtarget->hasXOP())) {
12367 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTDrr, RC: &X86::VR128RegClass, Op0, Op1);
12368 }
12369 return Register();
12370}
12371
12372Register fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12373 if (RetVT.SimpleTy != MVT::v8i32)
12374 return Register();
12375 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12376 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12377 }
12378 return Register();
12379}
12380
12381Register fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12382 if (RetVT.SimpleTy != MVT::v16i32)
12383 return Register();
12384 if ((Subtarget->hasAVX512())) {
12385 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12386 }
12387 return Register();
12388}
12389
12390Register fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12391 if (RetVT.SimpleTy != MVT::v2i64)
12392 return Register();
12393 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12394 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12395 }
12396 if ((Subtarget->hasXOP())) {
12397 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTQrr, RC: &X86::VR128RegClass, Op0, Op1);
12398 }
12399 return Register();
12400}
12401
12402Register fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12403 if (RetVT.SimpleTy != MVT::v4i64)
12404 return Register();
12405 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12406 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12407 }
12408 return Register();
12409}
12410
12411Register fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12412 if (RetVT.SimpleTy != MVT::v8i64)
12413 return Register();
12414 if ((Subtarget->hasAVX512())) {
12415 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12416 }
12417 return Register();
12418}
12419
12420Register fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12421 switch (VT.SimpleTy) {
12422 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op1);
12423 case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op1);
12424 case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op1);
12425 case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op1);
12426 case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op1);
12427 case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op1);
12428 case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op1);
12429 case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op1);
12430 case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op1);
12431 default: return Register();
12432 }
12433}
12434
12435// FastEmit functions for ISD::ROTR.
12436
12437Register fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12438 if (RetVT.SimpleTy != MVT::i8)
12439 return Register();
12440 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12441 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12442 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL_ND, RC: &X86::GR8RegClass, Op0);
12443 }
12444 if ((!Subtarget->hasNDD())) {
12445 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12446 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL, RC: &X86::GR8RegClass, Op0);
12447 }
12448 return Register();
12449}
12450
12451Register fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12452 if (RetVT.SimpleTy != MVT::v4i32)
12453 return Register();
12454 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12455 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12456 }
12457 return Register();
12458}
12459
12460Register fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12461 if (RetVT.SimpleTy != MVT::v8i32)
12462 return Register();
12463 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12464 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12465 }
12466 return Register();
12467}
12468
12469Register fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12470 if (RetVT.SimpleTy != MVT::v16i32)
12471 return Register();
12472 if ((Subtarget->hasAVX512())) {
12473 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12474 }
12475 return Register();
12476}
12477
12478Register fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12479 if (RetVT.SimpleTy != MVT::v2i64)
12480 return Register();
12481 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12482 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12483 }
12484 return Register();
12485}
12486
12487Register fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12488 if (RetVT.SimpleTy != MVT::v4i64)
12489 return Register();
12490 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12491 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12492 }
12493 return Register();
12494}
12495
12496Register fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12497 if (RetVT.SimpleTy != MVT::v8i64)
12498 return Register();
12499 if ((Subtarget->hasAVX512())) {
12500 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12501 }
12502 return Register();
12503}
12504
12505Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12506 switch (VT.SimpleTy) {
12507 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op1);
12508 case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op1);
12509 case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op1);
12510 case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op1);
12511 case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op1);
12512 case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op1);
12513 case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op1);
12514 default: return Register();
12515 }
12516}
12517
12518// FastEmit functions for ISD::SADDSAT.
12519
12520Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12521 if (RetVT.SimpleTy != MVT::v16i8)
12522 return Register();
12523 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12524 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12525 }
12526 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12527 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12528 }
12529 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12530 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12531 }
12532 return Register();
12533}
12534
12535Register fastEmit_ISD_SADDSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12536 if (RetVT.SimpleTy != MVT::v32i8)
12537 return Register();
12538 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12539 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12540 }
12541 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12542 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12543 }
12544 return Register();
12545}
12546
12547Register fastEmit_ISD_SADDSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12548 if (RetVT.SimpleTy != MVT::v64i8)
12549 return Register();
12550 if ((Subtarget->hasBWI())) {
12551 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12552 }
12553 return Register();
12554}
12555
12556Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12557 if (RetVT.SimpleTy != MVT::v8i16)
12558 return Register();
12559 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12560 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12561 }
12562 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12563 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12564 }
12565 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12566 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12567 }
12568 return Register();
12569}
12570
12571Register fastEmit_ISD_SADDSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12572 if (RetVT.SimpleTy != MVT::v16i16)
12573 return Register();
12574 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12575 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12576 }
12577 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12578 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12579 }
12580 return Register();
12581}
12582
12583Register fastEmit_ISD_SADDSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12584 if (RetVT.SimpleTy != MVT::v32i16)
12585 return Register();
12586 if ((Subtarget->hasBWI())) {
12587 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12588 }
12589 return Register();
12590}
12591
12592Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12593 switch (VT.SimpleTy) {
12594 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12595 case MVT::v32i8: return fastEmit_ISD_SADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
12596 case MVT::v64i8: return fastEmit_ISD_SADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
12597 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12598 case MVT::v16i16: return fastEmit_ISD_SADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
12599 case MVT::v32i16: return fastEmit_ISD_SADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
12600 default: return Register();
12601 }
12602}
12603
12604// FastEmit functions for ISD::SHL.
12605
12606Register fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12607 if (RetVT.SimpleTy != MVT::i8)
12608 return Register();
12609 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12610 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12611 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL_ND, RC: &X86::GR8RegClass, Op0);
12612 }
12613 if ((!Subtarget->hasNDD())) {
12614 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12615 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL, RC: &X86::GR8RegClass, Op0);
12616 }
12617 return Register();
12618}
12619
12620Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12621 switch (VT.SimpleTy) {
12622 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op1);
12623 default: return Register();
12624 }
12625}
12626
12627// FastEmit functions for ISD::SMAX.
12628
12629Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12630 if (RetVT.SimpleTy != MVT::v16i8)
12631 return Register();
12632 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12633 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12634 }
12635 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12636 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12637 }
12638 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12639 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12640 }
12641 return Register();
12642}
12643
12644Register fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12645 if (RetVT.SimpleTy != MVT::v32i8)
12646 return Register();
12647 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12648 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12649 }
12650 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12651 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12652 }
12653 return Register();
12654}
12655
12656Register fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12657 if (RetVT.SimpleTy != MVT::v64i8)
12658 return Register();
12659 if ((Subtarget->hasBWI())) {
12660 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12661 }
12662 return Register();
12663}
12664
12665Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12666 if (RetVT.SimpleTy != MVT::v8i16)
12667 return Register();
12668 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12669 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12670 }
12671 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12672 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12673 }
12674 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12675 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12676 }
12677 return Register();
12678}
12679
12680Register fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12681 if (RetVT.SimpleTy != MVT::v16i16)
12682 return Register();
12683 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12684 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12685 }
12686 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12687 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12688 }
12689 return Register();
12690}
12691
12692Register fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12693 if (RetVT.SimpleTy != MVT::v32i16)
12694 return Register();
12695 if ((Subtarget->hasBWI())) {
12696 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12697 }
12698 return Register();
12699}
12700
12701Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12702 if (RetVT.SimpleTy != MVT::v4i32)
12703 return Register();
12704 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12705 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12706 }
12707 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12708 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12709 }
12710 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12711 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12712 }
12713 return Register();
12714}
12715
12716Register fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12717 if (RetVT.SimpleTy != MVT::v8i32)
12718 return Register();
12719 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12720 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12721 }
12722 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12723 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12724 }
12725 return Register();
12726}
12727
12728Register fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12729 if (RetVT.SimpleTy != MVT::v16i32)
12730 return Register();
12731 if ((Subtarget->hasAVX512())) {
12732 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12733 }
12734 return Register();
12735}
12736
12737Register fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12738 if (RetVT.SimpleTy != MVT::v2i64)
12739 return Register();
12740 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12741 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12742 }
12743 return Register();
12744}
12745
12746Register fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12747 if (RetVT.SimpleTy != MVT::v4i64)
12748 return Register();
12749 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12750 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12751 }
12752 return Register();
12753}
12754
12755Register fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12756 if (RetVT.SimpleTy != MVT::v8i64)
12757 return Register();
12758 if ((Subtarget->hasAVX512())) {
12759 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12760 }
12761 return Register();
12762}
12763
12764Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12765 switch (VT.SimpleTy) {
12766 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12767 case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
12768 case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
12769 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12770 case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
12771 case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
12772 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12773 case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
12774 case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
12775 case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
12776 case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
12777 case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
12778 default: return Register();
12779 }
12780}
12781
12782// FastEmit functions for ISD::SMIN.
12783
12784Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12785 if (RetVT.SimpleTy != MVT::v16i8)
12786 return Register();
12787 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12788 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12789 }
12790 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12791 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12792 }
12793 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12794 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12795 }
12796 return Register();
12797}
12798
12799Register fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12800 if (RetVT.SimpleTy != MVT::v32i8)
12801 return Register();
12802 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12803 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12804 }
12805 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12806 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12807 }
12808 return Register();
12809}
12810
12811Register fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12812 if (RetVT.SimpleTy != MVT::v64i8)
12813 return Register();
12814 if ((Subtarget->hasBWI())) {
12815 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12816 }
12817 return Register();
12818}
12819
12820Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12821 if (RetVT.SimpleTy != MVT::v8i16)
12822 return Register();
12823 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12824 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12825 }
12826 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12827 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12828 }
12829 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12830 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12831 }
12832 return Register();
12833}
12834
12835Register fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12836 if (RetVT.SimpleTy != MVT::v16i16)
12837 return Register();
12838 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12839 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12840 }
12841 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12842 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12843 }
12844 return Register();
12845}
12846
12847Register fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12848 if (RetVT.SimpleTy != MVT::v32i16)
12849 return Register();
12850 if ((Subtarget->hasBWI())) {
12851 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12852 }
12853 return Register();
12854}
12855
12856Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12857 if (RetVT.SimpleTy != MVT::v4i32)
12858 return Register();
12859 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12860 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12861 }
12862 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12863 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12864 }
12865 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12866 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12867 }
12868 return Register();
12869}
12870
12871Register fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12872 if (RetVT.SimpleTy != MVT::v8i32)
12873 return Register();
12874 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12875 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12876 }
12877 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12878 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12879 }
12880 return Register();
12881}
12882
12883Register fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12884 if (RetVT.SimpleTy != MVT::v16i32)
12885 return Register();
12886 if ((Subtarget->hasAVX512())) {
12887 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12888 }
12889 return Register();
12890}
12891
12892Register fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12893 if (RetVT.SimpleTy != MVT::v2i64)
12894 return Register();
12895 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12896 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12897 }
12898 return Register();
12899}
12900
12901Register fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12902 if (RetVT.SimpleTy != MVT::v4i64)
12903 return Register();
12904 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12905 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12906 }
12907 return Register();
12908}
12909
12910Register fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12911 if (RetVT.SimpleTy != MVT::v8i64)
12912 return Register();
12913 if ((Subtarget->hasAVX512())) {
12914 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12915 }
12916 return Register();
12917}
12918
12919Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12920 switch (VT.SimpleTy) {
12921 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12922 case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
12923 case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
12924 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12925 case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
12926 case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
12927 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12928 case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
12929 case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
12930 case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
12931 case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
12932 case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
12933 default: return Register();
12934 }
12935}
12936
12937// FastEmit functions for ISD::SRA.
12938
12939Register fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12940 if (RetVT.SimpleTy != MVT::i8)
12941 return Register();
12942 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12943 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12944 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL_ND, RC: &X86::GR8RegClass, Op0);
12945 }
12946 if ((!Subtarget->hasNDD())) {
12947 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12948 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL, RC: &X86::GR8RegClass, Op0);
12949 }
12950 return Register();
12951}
12952
12953Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12954 switch (VT.SimpleTy) {
12955 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op1);
12956 default: return Register();
12957 }
12958}
12959
12960// FastEmit functions for ISD::SRL.
12961
12962Register fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12963 if (RetVT.SimpleTy != MVT::i8)
12964 return Register();
12965 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12966 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12967 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL_ND, RC: &X86::GR8RegClass, Op0);
12968 }
12969 if ((!Subtarget->hasNDD())) {
12970 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12971 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL, RC: &X86::GR8RegClass, Op0);
12972 }
12973 return Register();
12974}
12975
12976Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12977 switch (VT.SimpleTy) {
12978 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op1);
12979 default: return Register();
12980 }
12981}
12982
12983// FastEmit functions for ISD::SSUBSAT.
12984
12985Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12986 if (RetVT.SimpleTy != MVT::v16i8)
12987 return Register();
12988 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12989 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12990 }
12991 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12992 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12993 }
12994 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12995 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12996 }
12997 return Register();
12998}
12999
13000Register fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13001 if (RetVT.SimpleTy != MVT::v32i8)
13002 return Register();
13003 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13004 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13005 }
13006 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13007 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13008 }
13009 return Register();
13010}
13011
13012Register fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13013 if (RetVT.SimpleTy != MVT::v64i8)
13014 return Register();
13015 if ((Subtarget->hasBWI())) {
13016 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13017 }
13018 return Register();
13019}
13020
13021Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13022 if (RetVT.SimpleTy != MVT::v8i16)
13023 return Register();
13024 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13025 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13026 }
13027 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13028 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13029 }
13030 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13031 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13032 }
13033 return Register();
13034}
13035
13036Register fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13037 if (RetVT.SimpleTy != MVT::v16i16)
13038 return Register();
13039 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13040 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13041 }
13042 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13043 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13044 }
13045 return Register();
13046}
13047
13048Register fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13049 if (RetVT.SimpleTy != MVT::v32i16)
13050 return Register();
13051 if ((Subtarget->hasBWI())) {
13052 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13053 }
13054 return Register();
13055}
13056
13057Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13058 switch (VT.SimpleTy) {
13059 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13060 case MVT::v32i8: return fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
13061 case MVT::v64i8: return fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
13062 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13063 case MVT::v16i16: return fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
13064 case MVT::v32i16: return fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
13065 default: return Register();
13066 }
13067}
13068
13069// FastEmit functions for ISD::STRICT_FADD.
13070
13071Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13072 if (RetVT.SimpleTy != MVT::f16)
13073 return Register();
13074 if ((Subtarget->hasFP16())) {
13075 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13076 }
13077 return Register();
13078}
13079
13080Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13081 if (RetVT.SimpleTy != MVT::f32)
13082 return Register();
13083 if ((Subtarget->hasAVX512())) {
13084 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13085 }
13086 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13087 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13088 }
13089 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13090 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13091 }
13092 if ((!Subtarget->hasSSE1())) {
13093 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13094 }
13095 return Register();
13096}
13097
13098Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13099 if (RetVT.SimpleTy != MVT::f64)
13100 return Register();
13101 if ((Subtarget->hasAVX512())) {
13102 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13103 }
13104 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13105 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13106 }
13107 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13108 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13109 }
13110 if ((!Subtarget->hasSSE2())) {
13111 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13112 }
13113 return Register();
13114}
13115
13116Register fastEmit_ISD_STRICT_FADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13117 if (RetVT.SimpleTy != MVT::f80)
13118 return Register();
13119 if ((Subtarget->hasX87())) {
13120 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13121 }
13122 return Register();
13123}
13124
13125Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13126 if (RetVT.SimpleTy != MVT::v8f16)
13127 return Register();
13128 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13129 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13130 }
13131 return Register();
13132}
13133
13134Register fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13135 if (RetVT.SimpleTy != MVT::v16f16)
13136 return Register();
13137 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13138 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13139 }
13140 return Register();
13141}
13142
13143Register fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13144 if (RetVT.SimpleTy != MVT::v32f16)
13145 return Register();
13146 if ((Subtarget->hasFP16())) {
13147 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13148 }
13149 return Register();
13150}
13151
13152Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13153 if (RetVT.SimpleTy != MVT::v4f32)
13154 return Register();
13155 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13156 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13157 }
13158 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13159 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13160 }
13161 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13162 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13163 }
13164 return Register();
13165}
13166
13167Register fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13168 if (RetVT.SimpleTy != MVT::v8f32)
13169 return Register();
13170 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13171 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13172 }
13173 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13174 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13175 }
13176 return Register();
13177}
13178
13179Register fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13180 if (RetVT.SimpleTy != MVT::v16f32)
13181 return Register();
13182 if ((Subtarget->hasAVX512())) {
13183 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13184 }
13185 return Register();
13186}
13187
13188Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13189 if (RetVT.SimpleTy != MVT::v2f64)
13190 return Register();
13191 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13192 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13193 }
13194 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13195 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13196 }
13197 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13198 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13199 }
13200 return Register();
13201}
13202
13203Register fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13204 if (RetVT.SimpleTy != MVT::v4f64)
13205 return Register();
13206 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13207 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13208 }
13209 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13210 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13211 }
13212 return Register();
13213}
13214
13215Register fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13216 if (RetVT.SimpleTy != MVT::v8f64)
13217 return Register();
13218 if ((Subtarget->hasAVX512())) {
13219 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13220 }
13221 return Register();
13222}
13223
13224Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13225 switch (VT.SimpleTy) {
13226 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
13227 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
13228 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
13229 case MVT::f80: return fastEmit_ISD_STRICT_FADD_MVT_f80_rr(RetVT, Op0, Op1);
13230 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
13231 case MVT::v16f16: return fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
13232 case MVT::v32f16: return fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
13233 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
13234 case MVT::v8f32: return fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
13235 case MVT::v16f32: return fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
13236 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
13237 case MVT::v4f64: return fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
13238 case MVT::v8f64: return fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
13239 default: return Register();
13240 }
13241}
13242
13243// FastEmit functions for ISD::STRICT_FDIV.
13244
13245Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13246 if (RetVT.SimpleTy != MVT::f16)
13247 return Register();
13248 if ((Subtarget->hasFP16())) {
13249 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13250 }
13251 return Register();
13252}
13253
13254Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13255 if (RetVT.SimpleTy != MVT::f32)
13256 return Register();
13257 if ((Subtarget->hasAVX512())) {
13258 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13259 }
13260 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13261 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13262 }
13263 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13264 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13265 }
13266 if ((!Subtarget->hasSSE1())) {
13267 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13268 }
13269 return Register();
13270}
13271
13272Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13273 if (RetVT.SimpleTy != MVT::f64)
13274 return Register();
13275 if ((Subtarget->hasAVX512())) {
13276 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13277 }
13278 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13279 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13280 }
13281 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13282 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13283 }
13284 if ((!Subtarget->hasSSE2())) {
13285 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13286 }
13287 return Register();
13288}
13289
13290Register fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13291 if (RetVT.SimpleTy != MVT::f80)
13292 return Register();
13293 if ((Subtarget->hasX87())) {
13294 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13295 }
13296 return Register();
13297}
13298
13299Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13300 if (RetVT.SimpleTy != MVT::v8f16)
13301 return Register();
13302 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13303 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13304 }
13305 return Register();
13306}
13307
13308Register fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13309 if (RetVT.SimpleTy != MVT::v16f16)
13310 return Register();
13311 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13312 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13313 }
13314 return Register();
13315}
13316
13317Register fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13318 if (RetVT.SimpleTy != MVT::v32f16)
13319 return Register();
13320 if ((Subtarget->hasFP16())) {
13321 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13322 }
13323 return Register();
13324}
13325
13326Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13327 if (RetVT.SimpleTy != MVT::v4f32)
13328 return Register();
13329 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13330 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13331 }
13332 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13333 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13334 }
13335 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13336 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13337 }
13338 return Register();
13339}
13340
13341Register fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13342 if (RetVT.SimpleTy != MVT::v8f32)
13343 return Register();
13344 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13345 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13346 }
13347 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13348 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13349 }
13350 return Register();
13351}
13352
13353Register fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13354 if (RetVT.SimpleTy != MVT::v16f32)
13355 return Register();
13356 if ((Subtarget->hasAVX512())) {
13357 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13358 }
13359 return Register();
13360}
13361
13362Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13363 if (RetVT.SimpleTy != MVT::v2f64)
13364 return Register();
13365 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13366 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13367 }
13368 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13369 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13370 }
13371 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13372 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13373 }
13374 return Register();
13375}
13376
13377Register fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13378 if (RetVT.SimpleTy != MVT::v4f64)
13379 return Register();
13380 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13381 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13382 }
13383 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13384 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13385 }
13386 return Register();
13387}
13388
13389Register fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13390 if (RetVT.SimpleTy != MVT::v8f64)
13391 return Register();
13392 if ((Subtarget->hasAVX512())) {
13393 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13394 }
13395 return Register();
13396}
13397
13398Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13399 switch (VT.SimpleTy) {
13400 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
13401 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
13402 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
13403 case MVT::f80: return fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
13404 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
13405 case MVT::v16f16: return fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
13406 case MVT::v32f16: return fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
13407 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
13408 case MVT::v8f32: return fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
13409 case MVT::v16f32: return fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
13410 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
13411 case MVT::v4f64: return fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
13412 case MVT::v8f64: return fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
13413 default: return Register();
13414 }
13415}
13416
13417// FastEmit functions for ISD::STRICT_FMUL.
13418
13419Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13420 if (RetVT.SimpleTy != MVT::f16)
13421 return Register();
13422 if ((Subtarget->hasFP16())) {
13423 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13424 }
13425 return Register();
13426}
13427
13428Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13429 if (RetVT.SimpleTy != MVT::f32)
13430 return Register();
13431 if ((Subtarget->hasAVX512())) {
13432 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13433 }
13434 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13435 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13436 }
13437 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13438 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13439 }
13440 if ((!Subtarget->hasSSE1())) {
13441 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13442 }
13443 return Register();
13444}
13445
13446Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13447 if (RetVT.SimpleTy != MVT::f64)
13448 return Register();
13449 if ((Subtarget->hasAVX512())) {
13450 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13451 }
13452 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13453 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13454 }
13455 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13456 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13457 }
13458 if ((!Subtarget->hasSSE2())) {
13459 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13460 }
13461 return Register();
13462}
13463
13464Register fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13465 if (RetVT.SimpleTy != MVT::f80)
13466 return Register();
13467 if ((Subtarget->hasX87())) {
13468 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13469 }
13470 return Register();
13471}
13472
13473Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13474 if (RetVT.SimpleTy != MVT::v8f16)
13475 return Register();
13476 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13477 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13478 }
13479 return Register();
13480}
13481
13482Register fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13483 if (RetVT.SimpleTy != MVT::v16f16)
13484 return Register();
13485 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13486 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13487 }
13488 return Register();
13489}
13490
13491Register fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13492 if (RetVT.SimpleTy != MVT::v32f16)
13493 return Register();
13494 if ((Subtarget->hasFP16())) {
13495 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13496 }
13497 return Register();
13498}
13499
13500Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13501 if (RetVT.SimpleTy != MVT::v4f32)
13502 return Register();
13503 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13504 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13505 }
13506 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13507 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13508 }
13509 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13510 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13511 }
13512 return Register();
13513}
13514
13515Register fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13516 if (RetVT.SimpleTy != MVT::v8f32)
13517 return Register();
13518 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13519 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13520 }
13521 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13522 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13523 }
13524 return Register();
13525}
13526
13527Register fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13528 if (RetVT.SimpleTy != MVT::v16f32)
13529 return Register();
13530 if ((Subtarget->hasAVX512())) {
13531 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13532 }
13533 return Register();
13534}
13535
13536Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13537 if (RetVT.SimpleTy != MVT::v2f64)
13538 return Register();
13539 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13540 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13541 }
13542 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13543 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13544 }
13545 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13546 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13547 }
13548 return Register();
13549}
13550
13551Register fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13552 if (RetVT.SimpleTy != MVT::v4f64)
13553 return Register();
13554 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13555 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13556 }
13557 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13558 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13559 }
13560 return Register();
13561}
13562
13563Register fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13564 if (RetVT.SimpleTy != MVT::v8f64)
13565 return Register();
13566 if ((Subtarget->hasAVX512())) {
13567 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13568 }
13569 return Register();
13570}
13571
13572Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13573 switch (VT.SimpleTy) {
13574 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13575 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13576 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13577 case MVT::f80: return fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
13578 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13579 case MVT::v16f16: return fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
13580 case MVT::v32f16: return fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
13581 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13582 case MVT::v8f32: return fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
13583 case MVT::v16f32: return fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
13584 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13585 case MVT::v4f64: return fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
13586 case MVT::v8f64: return fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
13587 default: return Register();
13588 }
13589}
13590
13591// FastEmit functions for ISD::STRICT_FSUB.
13592
13593Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13594 if (RetVT.SimpleTy != MVT::f16)
13595 return Register();
13596 if ((Subtarget->hasFP16())) {
13597 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13598 }
13599 return Register();
13600}
13601
13602Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13603 if (RetVT.SimpleTy != MVT::f32)
13604 return Register();
13605 if ((Subtarget->hasAVX512())) {
13606 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13607 }
13608 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13609 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13610 }
13611 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13612 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13613 }
13614 if ((!Subtarget->hasSSE1())) {
13615 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13616 }
13617 return Register();
13618}
13619
13620Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13621 if (RetVT.SimpleTy != MVT::f64)
13622 return Register();
13623 if ((Subtarget->hasAVX512())) {
13624 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13625 }
13626 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13627 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13628 }
13629 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13630 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13631 }
13632 if ((!Subtarget->hasSSE2())) {
13633 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13634 }
13635 return Register();
13636}
13637
13638Register fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13639 if (RetVT.SimpleTy != MVT::f80)
13640 return Register();
13641 if ((Subtarget->hasX87())) {
13642 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13643 }
13644 return Register();
13645}
13646
13647Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13648 if (RetVT.SimpleTy != MVT::v8f16)
13649 return Register();
13650 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13651 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13652 }
13653 return Register();
13654}
13655
13656Register fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13657 if (RetVT.SimpleTy != MVT::v16f16)
13658 return Register();
13659 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13660 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13661 }
13662 return Register();
13663}
13664
13665Register fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13666 if (RetVT.SimpleTy != MVT::v32f16)
13667 return Register();
13668 if ((Subtarget->hasFP16())) {
13669 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13670 }
13671 return Register();
13672}
13673
13674Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13675 if (RetVT.SimpleTy != MVT::v4f32)
13676 return Register();
13677 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13678 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13679 }
13680 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13681 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13682 }
13683 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13684 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13685 }
13686 return Register();
13687}
13688
13689Register fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13690 if (RetVT.SimpleTy != MVT::v8f32)
13691 return Register();
13692 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13693 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13694 }
13695 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13696 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13697 }
13698 return Register();
13699}
13700
13701Register fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13702 if (RetVT.SimpleTy != MVT::v16f32)
13703 return Register();
13704 if ((Subtarget->hasAVX512())) {
13705 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13706 }
13707 return Register();
13708}
13709
13710Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13711 if (RetVT.SimpleTy != MVT::v2f64)
13712 return Register();
13713 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13714 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13715 }
13716 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13717 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13718 }
13719 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13720 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13721 }
13722 return Register();
13723}
13724
13725Register fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13726 if (RetVT.SimpleTy != MVT::v4f64)
13727 return Register();
13728 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13729 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13730 }
13731 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13732 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13733 }
13734 return Register();
13735}
13736
13737Register fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13738 if (RetVT.SimpleTy != MVT::v8f64)
13739 return Register();
13740 if ((Subtarget->hasAVX512())) {
13741 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13742 }
13743 return Register();
13744}
13745
13746Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13747 switch (VT.SimpleTy) {
13748 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13749 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13750 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13751 case MVT::f80: return fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
13752 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13753 case MVT::v16f16: return fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
13754 case MVT::v32f16: return fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
13755 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13756 case MVT::v8f32: return fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
13757 case MVT::v16f32: return fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
13758 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13759 case MVT::v4f64: return fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
13760 case MVT::v8f64: return fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
13761 default: return Register();
13762 }
13763}
13764
13765// FastEmit functions for ISD::SUB.
13766
13767Register fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13768 if (RetVT.SimpleTy != MVT::i8)
13769 return Register();
13770 if ((Subtarget->hasNDD())) {
13771 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
13772 }
13773 if ((!Subtarget->hasNDD())) {
13774 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr, RC: &X86::GR8RegClass, Op0, Op1);
13775 }
13776 return Register();
13777}
13778
13779Register fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
13780 if (RetVT.SimpleTy != MVT::i16)
13781 return Register();
13782 if ((Subtarget->hasNDD())) {
13783 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
13784 }
13785 if ((!Subtarget->hasNDD())) {
13786 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr, RC: &X86::GR16RegClass, Op0, Op1);
13787 }
13788 return Register();
13789}
13790
13791Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13792 if (RetVT.SimpleTy != MVT::i32)
13793 return Register();
13794 if ((Subtarget->hasNDD())) {
13795 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
13796 }
13797 if ((!Subtarget->hasNDD())) {
13798 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr, RC: &X86::GR32RegClass, Op0, Op1);
13799 }
13800 return Register();
13801}
13802
13803Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13804 if (RetVT.SimpleTy != MVT::i64)
13805 return Register();
13806 if ((Subtarget->hasNDD())) {
13807 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
13808 }
13809 if ((!Subtarget->hasNDD())) {
13810 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr, RC: &X86::GR64RegClass, Op0, Op1);
13811 }
13812 return Register();
13813}
13814
13815Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13816 if (RetVT.SimpleTy != MVT::v16i8)
13817 return Register();
13818 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13819 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13820 }
13821 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13822 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
13823 }
13824 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13825 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
13826 }
13827 return Register();
13828}
13829
13830Register fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13831 if (RetVT.SimpleTy != MVT::v32i8)
13832 return Register();
13833 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13834 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13835 }
13836 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13837 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13838 }
13839 return Register();
13840}
13841
13842Register fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13843 if (RetVT.SimpleTy != MVT::v64i8)
13844 return Register();
13845 if ((Subtarget->hasBWI())) {
13846 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13847 }
13848 return Register();
13849}
13850
13851Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13852 if (RetVT.SimpleTy != MVT::v8i16)
13853 return Register();
13854 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13855 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13856 }
13857 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13858 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
13859 }
13860 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13861 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
13862 }
13863 return Register();
13864}
13865
13866Register fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13867 if (RetVT.SimpleTy != MVT::v16i16)
13868 return Register();
13869 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13870 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13871 }
13872 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13873 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13874 }
13875 return Register();
13876}
13877
13878Register fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13879 if (RetVT.SimpleTy != MVT::v32i16)
13880 return Register();
13881 if ((Subtarget->hasBWI())) {
13882 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13883 }
13884 return Register();
13885}
13886
13887Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13888 if (RetVT.SimpleTy != MVT::v4i32)
13889 return Register();
13890 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13891 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13892 }
13893 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13894 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
13895 }
13896 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13897 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
13898 }
13899 return Register();
13900}
13901
13902Register fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
13903 if (RetVT.SimpleTy != MVT::v8i32)
13904 return Register();
13905 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13906 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13907 }
13908 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13909 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13910 }
13911 return Register();
13912}
13913
13914Register fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
13915 if (RetVT.SimpleTy != MVT::v16i32)
13916 return Register();
13917 if ((Subtarget->hasAVX512())) {
13918 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13919 }
13920 return Register();
13921}
13922
13923Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13924 if (RetVT.SimpleTy != MVT::v2i64)
13925 return Register();
13926 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13927 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13928 }
13929 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13930 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
13931 }
13932 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13933 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
13934 }
13935 return Register();
13936}
13937
13938Register fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
13939 if (RetVT.SimpleTy != MVT::v4i64)
13940 return Register();
13941 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13942 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13943 }
13944 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13945 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQYrr, RC: &X86::VR256RegClass, Op0, Op1);
13946 }
13947 return Register();
13948}
13949
13950Register fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
13951 if (RetVT.SimpleTy != MVT::v8i64)
13952 return Register();
13953 if ((Subtarget->hasAVX512())) {
13954 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13955 }
13956 return Register();
13957}
13958
13959Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13960 switch (VT.SimpleTy) {
13961 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op1);
13962 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op1);
13963 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13964 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13965 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13966 case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op1);
13967 case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op1);
13968 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13969 case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op1);
13970 case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op1);
13971 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13972 case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op1);
13973 case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op1);
13974 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13975 case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op1);
13976 case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op1);
13977 default: return Register();
13978 }
13979}
13980
13981// FastEmit functions for ISD::UADDSAT.
13982
13983Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13984 if (RetVT.SimpleTy != MVT::v16i8)
13985 return Register();
13986 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13987 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13988 }
13989 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13990 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13991 }
13992 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13993 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13994 }
13995 return Register();
13996}
13997
13998Register fastEmit_ISD_UADDSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13999 if (RetVT.SimpleTy != MVT::v32i8)
14000 return Register();
14001 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14002 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14003 }
14004 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14005 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14006 }
14007 return Register();
14008}
14009
14010Register fastEmit_ISD_UADDSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14011 if (RetVT.SimpleTy != MVT::v64i8)
14012 return Register();
14013 if ((Subtarget->hasBWI())) {
14014 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14015 }
14016 return Register();
14017}
14018
14019Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14020 if (RetVT.SimpleTy != MVT::v8i16)
14021 return Register();
14022 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14023 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14024 }
14025 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14026 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14027 }
14028 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14029 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14030 }
14031 return Register();
14032}
14033
14034Register fastEmit_ISD_UADDSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14035 if (RetVT.SimpleTy != MVT::v16i16)
14036 return Register();
14037 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14038 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14039 }
14040 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14041 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14042 }
14043 return Register();
14044}
14045
14046Register fastEmit_ISD_UADDSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14047 if (RetVT.SimpleTy != MVT::v32i16)
14048 return Register();
14049 if ((Subtarget->hasBWI())) {
14050 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14051 }
14052 return Register();
14053}
14054
14055Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14056 switch (VT.SimpleTy) {
14057 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14058 case MVT::v32i8: return fastEmit_ISD_UADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
14059 case MVT::v64i8: return fastEmit_ISD_UADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
14060 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14061 case MVT::v16i16: return fastEmit_ISD_UADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
14062 case MVT::v32i16: return fastEmit_ISD_UADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
14063 default: return Register();
14064 }
14065}
14066
14067// FastEmit functions for ISD::UMAX.
14068
14069Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14070 if (RetVT.SimpleTy != MVT::v16i8)
14071 return Register();
14072 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14073 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14074 }
14075 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14076 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14077 }
14078 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14079 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14080 }
14081 return Register();
14082}
14083
14084Register fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14085 if (RetVT.SimpleTy != MVT::v32i8)
14086 return Register();
14087 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14088 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14089 }
14090 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14091 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14092 }
14093 return Register();
14094}
14095
14096Register fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14097 if (RetVT.SimpleTy != MVT::v64i8)
14098 return Register();
14099 if ((Subtarget->hasBWI())) {
14100 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14101 }
14102 return Register();
14103}
14104
14105Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14106 if (RetVT.SimpleTy != MVT::v8i16)
14107 return Register();
14108 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14109 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14110 }
14111 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14112 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14113 }
14114 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14115 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14116 }
14117 return Register();
14118}
14119
14120Register fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14121 if (RetVT.SimpleTy != MVT::v16i16)
14122 return Register();
14123 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14124 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14125 }
14126 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14127 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14128 }
14129 return Register();
14130}
14131
14132Register fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14133 if (RetVT.SimpleTy != MVT::v32i16)
14134 return Register();
14135 if ((Subtarget->hasBWI())) {
14136 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14137 }
14138 return Register();
14139}
14140
14141Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14142 if (RetVT.SimpleTy != MVT::v4i32)
14143 return Register();
14144 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14145 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14146 }
14147 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14148 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14149 }
14150 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14151 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14152 }
14153 return Register();
14154}
14155
14156Register fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14157 if (RetVT.SimpleTy != MVT::v8i32)
14158 return Register();
14159 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14160 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14161 }
14162 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14163 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14164 }
14165 return Register();
14166}
14167
14168Register fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14169 if (RetVT.SimpleTy != MVT::v16i32)
14170 return Register();
14171 if ((Subtarget->hasAVX512())) {
14172 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14173 }
14174 return Register();
14175}
14176
14177Register fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14178 if (RetVT.SimpleTy != MVT::v2i64)
14179 return Register();
14180 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14181 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14182 }
14183 return Register();
14184}
14185
14186Register fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14187 if (RetVT.SimpleTy != MVT::v4i64)
14188 return Register();
14189 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14190 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14191 }
14192 return Register();
14193}
14194
14195Register fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14196 if (RetVT.SimpleTy != MVT::v8i64)
14197 return Register();
14198 if ((Subtarget->hasAVX512())) {
14199 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14200 }
14201 return Register();
14202}
14203
14204Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14205 switch (VT.SimpleTy) {
14206 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
14207 case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
14208 case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
14209 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
14210 case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
14211 case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
14212 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
14213 case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
14214 case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
14215 case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
14216 case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
14217 case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
14218 default: return Register();
14219 }
14220}
14221
14222// FastEmit functions for ISD::UMIN.
14223
14224Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14225 if (RetVT.SimpleTy != MVT::v16i8)
14226 return Register();
14227 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14228 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14229 }
14230 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14231 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14232 }
14233 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14234 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14235 }
14236 return Register();
14237}
14238
14239Register fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14240 if (RetVT.SimpleTy != MVT::v32i8)
14241 return Register();
14242 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14243 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14244 }
14245 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14246 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14247 }
14248 return Register();
14249}
14250
14251Register fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14252 if (RetVT.SimpleTy != MVT::v64i8)
14253 return Register();
14254 if ((Subtarget->hasBWI())) {
14255 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14256 }
14257 return Register();
14258}
14259
14260Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14261 if (RetVT.SimpleTy != MVT::v8i16)
14262 return Register();
14263 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14264 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14265 }
14266 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14267 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14268 }
14269 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14270 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14271 }
14272 return Register();
14273}
14274
14275Register fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14276 if (RetVT.SimpleTy != MVT::v16i16)
14277 return Register();
14278 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14279 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14280 }
14281 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14282 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14283 }
14284 return Register();
14285}
14286
14287Register fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14288 if (RetVT.SimpleTy != MVT::v32i16)
14289 return Register();
14290 if ((Subtarget->hasBWI())) {
14291 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14292 }
14293 return Register();
14294}
14295
14296Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14297 if (RetVT.SimpleTy != MVT::v4i32)
14298 return Register();
14299 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14300 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14301 }
14302 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14303 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14304 }
14305 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14306 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14307 }
14308 return Register();
14309}
14310
14311Register fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14312 if (RetVT.SimpleTy != MVT::v8i32)
14313 return Register();
14314 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14315 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14316 }
14317 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14318 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14319 }
14320 return Register();
14321}
14322
14323Register fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14324 if (RetVT.SimpleTy != MVT::v16i32)
14325 return Register();
14326 if ((Subtarget->hasAVX512())) {
14327 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14328 }
14329 return Register();
14330}
14331
14332Register fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14333 if (RetVT.SimpleTy != MVT::v2i64)
14334 return Register();
14335 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14336 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14337 }
14338 return Register();
14339}
14340
14341Register fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14342 if (RetVT.SimpleTy != MVT::v4i64)
14343 return Register();
14344 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14345 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14346 }
14347 return Register();
14348}
14349
14350Register fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14351 if (RetVT.SimpleTy != MVT::v8i64)
14352 return Register();
14353 if ((Subtarget->hasAVX512())) {
14354 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14355 }
14356 return Register();
14357}
14358
14359Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14360 switch (VT.SimpleTy) {
14361 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
14362 case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
14363 case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
14364 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
14365 case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
14366 case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
14367 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
14368 case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
14369 case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
14370 case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
14371 case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
14372 case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
14373 default: return Register();
14374 }
14375}
14376
14377// FastEmit functions for ISD::USUBSAT.
14378
14379Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14380 if (RetVT.SimpleTy != MVT::v16i8)
14381 return Register();
14382 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14383 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14384 }
14385 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14386 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14387 }
14388 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14389 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14390 }
14391 return Register();
14392}
14393
14394Register fastEmit_ISD_USUBSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14395 if (RetVT.SimpleTy != MVT::v32i8)
14396 return Register();
14397 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14398 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14399 }
14400 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14401 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14402 }
14403 return Register();
14404}
14405
14406Register fastEmit_ISD_USUBSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14407 if (RetVT.SimpleTy != MVT::v64i8)
14408 return Register();
14409 if ((Subtarget->hasBWI())) {
14410 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14411 }
14412 return Register();
14413}
14414
14415Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14416 if (RetVT.SimpleTy != MVT::v8i16)
14417 return Register();
14418 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14419 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14420 }
14421 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14422 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14423 }
14424 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14425 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14426 }
14427 return Register();
14428}
14429
14430Register fastEmit_ISD_USUBSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14431 if (RetVT.SimpleTy != MVT::v16i16)
14432 return Register();
14433 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14434 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14435 }
14436 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14437 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14438 }
14439 return Register();
14440}
14441
14442Register fastEmit_ISD_USUBSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14443 if (RetVT.SimpleTy != MVT::v32i16)
14444 return Register();
14445 if ((Subtarget->hasBWI())) {
14446 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14447 }
14448 return Register();
14449}
14450
14451Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14452 switch (VT.SimpleTy) {
14453 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14454 case MVT::v32i8: return fastEmit_ISD_USUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
14455 case MVT::v64i8: return fastEmit_ISD_USUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
14456 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14457 case MVT::v16i16: return fastEmit_ISD_USUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
14458 case MVT::v32i16: return fastEmit_ISD_USUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
14459 default: return Register();
14460 }
14461}
14462
14463// FastEmit functions for ISD::XOR.
14464
14465Register fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
14466 if (RetVT.SimpleTy != MVT::i8)
14467 return Register();
14468 if ((Subtarget->hasNDD())) {
14469 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
14470 }
14471 if ((!Subtarget->hasNDD())) {
14472 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr, RC: &X86::GR8RegClass, Op0, Op1);
14473 }
14474 return Register();
14475}
14476
14477Register fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
14478 if (RetVT.SimpleTy != MVT::i16)
14479 return Register();
14480 if ((Subtarget->hasNDD())) {
14481 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
14482 }
14483 if ((!Subtarget->hasNDD())) {
14484 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr, RC: &X86::GR16RegClass, Op0, Op1);
14485 }
14486 return Register();
14487}
14488
14489Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14490 if (RetVT.SimpleTy != MVT::i32)
14491 return Register();
14492 if ((Subtarget->hasNDD())) {
14493 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
14494 }
14495 if ((!Subtarget->hasNDD())) {
14496 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr, RC: &X86::GR32RegClass, Op0, Op1);
14497 }
14498 return Register();
14499}
14500
14501Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14502 if (RetVT.SimpleTy != MVT::i64)
14503 return Register();
14504 if ((Subtarget->hasNDD())) {
14505 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
14506 }
14507 if ((!Subtarget->hasNDD())) {
14508 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr, RC: &X86::GR64RegClass, Op0, Op1);
14509 }
14510 return Register();
14511}
14512
14513Register fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
14514 if (RetVT.SimpleTy != MVT::v8i1)
14515 return Register();
14516 if ((Subtarget->hasDQI())) {
14517 return fastEmitInst_rr(MachineInstOpcode: X86::KXORBkk, RC: &X86::VK8RegClass, Op0, Op1);
14518 }
14519 return Register();
14520}
14521
14522Register fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
14523 if (RetVT.SimpleTy != MVT::v16i1)
14524 return Register();
14525 if ((Subtarget->hasAVX512())) {
14526 return fastEmitInst_rr(MachineInstOpcode: X86::KXORWkk, RC: &X86::VK16RegClass, Op0, Op1);
14527 }
14528 return Register();
14529}
14530
14531Register fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
14532 if (RetVT.SimpleTy != MVT::v32i1)
14533 return Register();
14534 if ((Subtarget->hasBWI())) {
14535 return fastEmitInst_rr(MachineInstOpcode: X86::KXORDkk, RC: &X86::VK32RegClass, Op0, Op1);
14536 }
14537 return Register();
14538}
14539
14540Register fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
14541 if (RetVT.SimpleTy != MVT::v64i1)
14542 return Register();
14543 if ((Subtarget->hasBWI())) {
14544 return fastEmitInst_rr(MachineInstOpcode: X86::KXORQkk, RC: &X86::VK64RegClass, Op0, Op1);
14545 }
14546 return Register();
14547}
14548
14549Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14550 if (RetVT.SimpleTy != MVT::v16i8)
14551 return Register();
14552 if ((Subtarget->hasVLX())) {
14553 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14554 }
14555 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14556 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14557 }
14558 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14559 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14560 }
14561 return Register();
14562}
14563
14564Register fastEmit_ISD_XOR_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14565 if (RetVT.SimpleTy != MVT::v32i8)
14566 return Register();
14567 if ((Subtarget->hasVLX())) {
14568 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14569 }
14570 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14571 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14572 }
14573 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14574 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14575 }
14576 return Register();
14577}
14578
14579Register fastEmit_ISD_XOR_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14580 if (RetVT.SimpleTy != MVT::v64i8)
14581 return Register();
14582 if ((Subtarget->hasAVX512())) {
14583 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14584 }
14585 return Register();
14586}
14587
14588Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14589 if (RetVT.SimpleTy != MVT::v8i16)
14590 return Register();
14591 if ((Subtarget->hasVLX())) {
14592 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14593 }
14594 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14595 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14596 }
14597 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14598 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14599 }
14600 return Register();
14601}
14602
14603Register fastEmit_ISD_XOR_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14604 if (RetVT.SimpleTy != MVT::v16i16)
14605 return Register();
14606 if ((Subtarget->hasVLX())) {
14607 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14608 }
14609 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14610 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14611 }
14612 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14613 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14614 }
14615 return Register();
14616}
14617
14618Register fastEmit_ISD_XOR_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14619 if (RetVT.SimpleTy != MVT::v32i16)
14620 return Register();
14621 if ((Subtarget->hasAVX512())) {
14622 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14623 }
14624 return Register();
14625}
14626
14627Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14628 if (RetVT.SimpleTy != MVT::v4i32)
14629 return Register();
14630 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14631 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14632 }
14633 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14634 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14635 }
14636 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14637 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14638 }
14639 return Register();
14640}
14641
14642Register fastEmit_ISD_XOR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14643 if (RetVT.SimpleTy != MVT::v8i32)
14644 return Register();
14645 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14646 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14647 }
14648 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14649 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14650 }
14651 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14652 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14653 }
14654 return Register();
14655}
14656
14657Register fastEmit_ISD_XOR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14658 if (RetVT.SimpleTy != MVT::v16i32)
14659 return Register();
14660 if ((Subtarget->hasAVX512())) {
14661 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14662 }
14663 return Register();
14664}
14665
14666Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14667 if (RetVT.SimpleTy != MVT::v2i64)
14668 return Register();
14669 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14670 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14671 }
14672 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14673 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14674 }
14675 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14676 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14677 }
14678 return Register();
14679}
14680
14681Register fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14682 if (RetVT.SimpleTy != MVT::v4i64)
14683 return Register();
14684 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14685 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14686 }
14687 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14688 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14689 }
14690 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14691 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14692 }
14693 return Register();
14694}
14695
14696Register fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14697 if (RetVT.SimpleTy != MVT::v8i64)
14698 return Register();
14699 if ((Subtarget->hasAVX512())) {
14700 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14701 }
14702 return Register();
14703}
14704
14705Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14706 switch (VT.SimpleTy) {
14707 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op1);
14708 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op1);
14709 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14710 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14711 case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op1);
14712 case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op1);
14713 case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op1);
14714 case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op1);
14715 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14716 case MVT::v32i8: return fastEmit_ISD_XOR_MVT_v32i8_rr(RetVT, Op0, Op1);
14717 case MVT::v64i8: return fastEmit_ISD_XOR_MVT_v64i8_rr(RetVT, Op0, Op1);
14718 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14719 case MVT::v16i16: return fastEmit_ISD_XOR_MVT_v16i16_rr(RetVT, Op0, Op1);
14720 case MVT::v32i16: return fastEmit_ISD_XOR_MVT_v32i16_rr(RetVT, Op0, Op1);
14721 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14722 case MVT::v8i32: return fastEmit_ISD_XOR_MVT_v8i32_rr(RetVT, Op0, Op1);
14723 case MVT::v16i32: return fastEmit_ISD_XOR_MVT_v16i32_rr(RetVT, Op0, Op1);
14724 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14725 case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op1);
14726 case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op1);
14727 default: return Register();
14728 }
14729}
14730
14731// FastEmit functions for X86ISD::ADDSUB.
14732
14733Register fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
14734 if (RetVT.SimpleTy != MVT::v4f32)
14735 return Register();
14736 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
14737 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14738 }
14739 if ((Subtarget->hasAVX())) {
14740 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14741 }
14742 return Register();
14743}
14744
14745Register fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
14746 if (RetVT.SimpleTy != MVT::v8f32)
14747 return Register();
14748 if ((Subtarget->hasAVX())) {
14749 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14750 }
14751 return Register();
14752}
14753
14754Register fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14755 if (RetVT.SimpleTy != MVT::v2f64)
14756 return Register();
14757 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
14758 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14759 }
14760 if ((Subtarget->hasAVX())) {
14761 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14762 }
14763 return Register();
14764}
14765
14766Register fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14767 if (RetVT.SimpleTy != MVT::v4f64)
14768 return Register();
14769 if ((Subtarget->hasAVX())) {
14770 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14771 }
14772 return Register();
14773}
14774
14775Register fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14776 switch (VT.SimpleTy) {
14777 case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
14778 case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
14779 case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
14780 case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
14781 default: return Register();
14782 }
14783}
14784
14785// FastEmit functions for X86ISD::ANDNP.
14786
14787Register fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14788 if (RetVT.SimpleTy != MVT::v16i8)
14789 return Register();
14790 if ((Subtarget->hasVLX())) {
14791 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14792 }
14793 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14794 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14795 }
14796 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14797 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14798 }
14799 return Register();
14800}
14801
14802Register fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14803 if (RetVT.SimpleTy != MVT::v32i8)
14804 return Register();
14805 if ((Subtarget->hasVLX())) {
14806 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14807 }
14808 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14809 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14810 }
14811 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14812 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14813 }
14814 return Register();
14815}
14816
14817Register fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14818 if (RetVT.SimpleTy != MVT::v64i8)
14819 return Register();
14820 if ((Subtarget->hasAVX512())) {
14821 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14822 }
14823 return Register();
14824}
14825
14826Register fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14827 if (RetVT.SimpleTy != MVT::v8i16)
14828 return Register();
14829 if ((Subtarget->hasVLX())) {
14830 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14831 }
14832 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14833 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14834 }
14835 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14836 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14837 }
14838 return Register();
14839}
14840
14841Register fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14842 if (RetVT.SimpleTy != MVT::v16i16)
14843 return Register();
14844 if ((Subtarget->hasVLX())) {
14845 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14846 }
14847 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14848 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14849 }
14850 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14851 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14852 }
14853 return Register();
14854}
14855
14856Register fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14857 if (RetVT.SimpleTy != MVT::v32i16)
14858 return Register();
14859 if ((Subtarget->hasAVX512())) {
14860 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14861 }
14862 return Register();
14863}
14864
14865Register fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14866 if (RetVT.SimpleTy != MVT::v4i32)
14867 return Register();
14868 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14869 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14870 }
14871 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14872 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14873 }
14874 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14875 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14876 }
14877 return Register();
14878}
14879
14880Register fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14881 if (RetVT.SimpleTy != MVT::v8i32)
14882 return Register();
14883 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14884 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14885 }
14886 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14887 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14888 }
14889 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14890 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14891 }
14892 return Register();
14893}
14894
14895Register fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14896 if (RetVT.SimpleTy != MVT::v16i32)
14897 return Register();
14898 if ((Subtarget->hasAVX512())) {
14899 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14900 }
14901 return Register();
14902}
14903
14904Register fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14905 if (RetVT.SimpleTy != MVT::v2i64)
14906 return Register();
14907 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14908 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14909 }
14910 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14911 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14912 }
14913 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14914 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14915 }
14916 return Register();
14917}
14918
14919Register fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14920 if (RetVT.SimpleTy != MVT::v4i64)
14921 return Register();
14922 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14923 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14924 }
14925 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14926 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14927 }
14928 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14929 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14930 }
14931 return Register();
14932}
14933
14934Register fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14935 if (RetVT.SimpleTy != MVT::v8i64)
14936 return Register();
14937 if ((Subtarget->hasAVX512())) {
14938 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14939 }
14940 return Register();
14941}
14942
14943Register fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14944 switch (VT.SimpleTy) {
14945 case MVT::v16i8: return fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(RetVT, Op0, Op1);
14946 case MVT::v32i8: return fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(RetVT, Op0, Op1);
14947 case MVT::v64i8: return fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(RetVT, Op0, Op1);
14948 case MVT::v8i16: return fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(RetVT, Op0, Op1);
14949 case MVT::v16i16: return fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(RetVT, Op0, Op1);
14950 case MVT::v32i16: return fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(RetVT, Op0, Op1);
14951 case MVT::v4i32: return fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(RetVT, Op0, Op1);
14952 case MVT::v8i32: return fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(RetVT, Op0, Op1);
14953 case MVT::v16i32: return fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(RetVT, Op0, Op1);
14954 case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op1);
14955 case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op1);
14956 case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op1);
14957 default: return Register();
14958 }
14959}
14960
14961// FastEmit functions for X86ISD::BEXTR.
14962
14963Register fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14964 if (RetVT.SimpleTy != MVT::i32)
14965 return Register();
14966 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
14967 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
14968 }
14969 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
14970 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr, RC: &X86::GR32RegClass, Op0, Op1);
14971 }
14972 return Register();
14973}
14974
14975Register fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14976 if (RetVT.SimpleTy != MVT::i64)
14977 return Register();
14978 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
14979 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
14980 }
14981 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
14982 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr, RC: &X86::GR64RegClass, Op0, Op1);
14983 }
14984 return Register();
14985}
14986
14987Register fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14988 switch (VT.SimpleTy) {
14989 case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op1);
14990 case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op1);
14991 default: return Register();
14992 }
14993}
14994
14995// FastEmit functions for X86ISD::BT.
14996
14997Register fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
14998 if (RetVT.SimpleTy != MVT::i32)
14999 return Register();
15000 return fastEmitInst_rr(MachineInstOpcode: X86::BT16rr, RC: &X86::GR16RegClass, Op0, Op1);
15001}
15002
15003Register fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15004 if (RetVT.SimpleTy != MVT::i32)
15005 return Register();
15006 return fastEmitInst_rr(MachineInstOpcode: X86::BT32rr, RC: &X86::GR32RegClass, Op0, Op1);
15007}
15008
15009Register fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15010 if (RetVT.SimpleTy != MVT::i32)
15011 return Register();
15012 return fastEmitInst_rr(MachineInstOpcode: X86::BT64rr, RC: &X86::GR64RegClass, Op0, Op1);
15013}
15014
15015Register fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15016 switch (VT.SimpleTy) {
15017 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op1);
15018 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op1);
15019 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op1);
15020 default: return Register();
15021 }
15022}
15023
15024// FastEmit functions for X86ISD::BZHI.
15025
15026Register fastEmit_X86ISD_BZHI_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15027 if (RetVT.SimpleTy != MVT::i32)
15028 return Register();
15029 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15030 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
15031 }
15032 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15033 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr, RC: &X86::GR32RegClass, Op0, Op1);
15034 }
15035 return Register();
15036}
15037
15038Register fastEmit_X86ISD_BZHI_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15039 if (RetVT.SimpleTy != MVT::i64)
15040 return Register();
15041 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15042 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
15043 }
15044 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15045 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr, RC: &X86::GR64RegClass, Op0, Op1);
15046 }
15047 return Register();
15048}
15049
15050Register fastEmit_X86ISD_BZHI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15051 switch (VT.SimpleTy) {
15052 case MVT::i32: return fastEmit_X86ISD_BZHI_MVT_i32_rr(RetVT, Op0, Op1);
15053 case MVT::i64: return fastEmit_X86ISD_BZHI_MVT_i64_rr(RetVT, Op0, Op1);
15054 default: return Register();
15055 }
15056}
15057
15058// FastEmit functions for X86ISD::CMP.
15059
15060Register fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
15061 if (RetVT.SimpleTy != MVT::i32)
15062 return Register();
15063 return fastEmitInst_rr(MachineInstOpcode: X86::CMP8rr, RC: &X86::GR8RegClass, Op0, Op1);
15064}
15065
15066Register fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
15067 if (RetVT.SimpleTy != MVT::i32)
15068 return Register();
15069 return fastEmitInst_rr(MachineInstOpcode: X86::CMP16rr, RC: &X86::GR16RegClass, Op0, Op1);
15070}
15071
15072Register fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15073 if (RetVT.SimpleTy != MVT::i32)
15074 return Register();
15075 return fastEmitInst_rr(MachineInstOpcode: X86::CMP32rr, RC: &X86::GR32RegClass, Op0, Op1);
15076}
15077
15078Register fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15079 if (RetVT.SimpleTy != MVT::i32)
15080 return Register();
15081 return fastEmitInst_rr(MachineInstOpcode: X86::CMP64rr, RC: &X86::GR64RegClass, Op0, Op1);
15082}
15083
15084Register fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15085 switch (VT.SimpleTy) {
15086 case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op1);
15087 case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op1);
15088 case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
15089 case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op1);
15090 default: return Register();
15091 }
15092}
15093
15094// FastEmit functions for X86ISD::COMI.
15095
15096Register fastEmit_X86ISD_COMI_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15097 if (RetVT.SimpleTy != MVT::i32)
15098 return Register();
15099 if ((Subtarget->hasFP16())) {
15100 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15101 }
15102 return Register();
15103}
15104
15105Register fastEmit_X86ISD_COMI_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
15106 if (RetVT.SimpleTy != MVT::i32)
15107 return Register();
15108 if ((Subtarget->hasAVX10_2())) {
15109 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISBF16Zrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15110 }
15111 return Register();
15112}
15113
15114Register fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15115 if (RetVT.SimpleTy != MVT::i32)
15116 return Register();
15117 if ((Subtarget->hasAVX512())) {
15118 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15119 }
15120 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15121 return fastEmitInst_rr(MachineInstOpcode: X86::COMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15122 }
15123 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15124 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15125 }
15126 return Register();
15127}
15128
15129Register fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15130 if (RetVT.SimpleTy != MVT::i32)
15131 return Register();
15132 if ((Subtarget->hasAVX512())) {
15133 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15134 }
15135 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15136 return fastEmitInst_rr(MachineInstOpcode: X86::COMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15137 }
15138 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15139 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15140 }
15141 return Register();
15142}
15143
15144Register fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15145 switch (VT.SimpleTy) {
15146 case MVT::v8f16: return fastEmit_X86ISD_COMI_MVT_v8f16_rr(RetVT, Op0, Op1);
15147 case MVT::v8bf16: return fastEmit_X86ISD_COMI_MVT_v8bf16_rr(RetVT, Op0, Op1);
15148 case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op1);
15149 case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op1);
15150 default: return Register();
15151 }
15152}
15153
15154// FastEmit functions for X86ISD::COMX.
15155
15156Register fastEmit_X86ISD_COMX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15157 if (RetVT.SimpleTy != MVT::i32)
15158 return Register();
15159 if ((Subtarget->hasAVX10_2())) {
15160 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15161 }
15162 return Register();
15163}
15164
15165Register fastEmit_X86ISD_COMX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15166 if (RetVT.SimpleTy != MVT::i32)
15167 return Register();
15168 if ((Subtarget->hasAVX10_2())) {
15169 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15170 }
15171 return Register();
15172}
15173
15174Register fastEmit_X86ISD_COMX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15175 if (RetVT.SimpleTy != MVT::i32)
15176 return Register();
15177 if ((Subtarget->hasAVX10_2())) {
15178 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15179 }
15180 return Register();
15181}
15182
15183Register fastEmit_X86ISD_COMX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15184 switch (VT.SimpleTy) {
15185 case MVT::v8f16: return fastEmit_X86ISD_COMX_MVT_v8f16_rr(RetVT, Op0, Op1);
15186 case MVT::v4f32: return fastEmit_X86ISD_COMX_MVT_v4f32_rr(RetVT, Op0, Op1);
15187 case MVT::v2f64: return fastEmit_X86ISD_COMX_MVT_v2f64_rr(RetVT, Op0, Op1);
15188 default: return Register();
15189 }
15190}
15191
15192// FastEmit functions for X86ISD::FADDS.
15193
15194Register fastEmit_X86ISD_FADDS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15195 if (RetVT.SimpleTy != MVT::v8f16)
15196 return Register();
15197 if ((Subtarget->hasFP16())) {
15198 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15199 }
15200 return Register();
15201}
15202
15203Register fastEmit_X86ISD_FADDS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15204 if (RetVT.SimpleTy != MVT::v4f32)
15205 return Register();
15206 if ((Subtarget->hasAVX512())) {
15207 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15208 }
15209 return Register();
15210}
15211
15212Register fastEmit_X86ISD_FADDS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15213 if (RetVT.SimpleTy != MVT::v2f64)
15214 return Register();
15215 if ((Subtarget->hasAVX512())) {
15216 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15217 }
15218 return Register();
15219}
15220
15221Register fastEmit_X86ISD_FADDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15222 switch (VT.SimpleTy) {
15223 case MVT::v8f16: return fastEmit_X86ISD_FADDS_MVT_v8f16_rr(RetVT, Op0, Op1);
15224 case MVT::v4f32: return fastEmit_X86ISD_FADDS_MVT_v4f32_rr(RetVT, Op0, Op1);
15225 case MVT::v2f64: return fastEmit_X86ISD_FADDS_MVT_v2f64_rr(RetVT, Op0, Op1);
15226 default: return Register();
15227 }
15228}
15229
15230// FastEmit functions for X86ISD::FAND.
15231
15232Register fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
15233 if (RetVT.SimpleTy != MVT::f128)
15234 return Register();
15235 if ((Subtarget->hasVLX())) {
15236 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15237 }
15238 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15239 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15240 }
15241 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15242 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15243 }
15244 return Register();
15245}
15246
15247Register fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15248 if (RetVT.SimpleTy != MVT::v4f32)
15249 return Register();
15250 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15251}
15252
15253Register fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15254 switch (VT.SimpleTy) {
15255 case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op1);
15256 case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op1);
15257 default: return Register();
15258 }
15259}
15260
15261// FastEmit functions for X86ISD::FANDN.
15262
15263Register fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15264 if (RetVT.SimpleTy != MVT::v4f32)
15265 return Register();
15266 return fastEmitInst_rr(MachineInstOpcode: X86::ANDNPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15267}
15268
15269Register fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15270 switch (VT.SimpleTy) {
15271 case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op1);
15272 default: return Register();
15273 }
15274}
15275
15276// FastEmit functions for X86ISD::FCMP.
15277
15278Register fastEmit_X86ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
15279 if (RetVT.SimpleTy != MVT::i32)
15280 return Register();
15281 if ((Subtarget->hasFP16())) {
15282 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
15283 }
15284 return Register();
15285}
15286
15287Register fastEmit_X86ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
15288 if (RetVT.SimpleTy != MVT::i32)
15289 return Register();
15290 if ((Subtarget->hasAVX512())) {
15291 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
15292 }
15293 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15294 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
15295 }
15296 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15297 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
15298 }
15299 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
15300 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
15301 }
15302 return Register();
15303}
15304
15305Register fastEmit_X86ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
15306 if (RetVT.SimpleTy != MVT::i32)
15307 return Register();
15308 if ((Subtarget->hasAVX512())) {
15309 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
15310 }
15311 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15312 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
15313 }
15314 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15315 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
15316 }
15317 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
15318 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
15319 }
15320 return Register();
15321}
15322
15323Register fastEmit_X86ISD_FCMP_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
15324 if (RetVT.SimpleTy != MVT::i32)
15325 return Register();
15326 if ((Subtarget->canUseCMOV())) {
15327 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
15328 }
15329 return Register();
15330}
15331
15332Register fastEmit_X86ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15333 switch (VT.SimpleTy) {
15334 case MVT::f16: return fastEmit_X86ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
15335 case MVT::f32: return fastEmit_X86ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
15336 case MVT::f64: return fastEmit_X86ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
15337 case MVT::f80: return fastEmit_X86ISD_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
15338 default: return Register();
15339 }
15340}
15341
15342// FastEmit functions for X86ISD::FDIVS.
15343
15344Register fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15345 if (RetVT.SimpleTy != MVT::v8f16)
15346 return Register();
15347 if ((Subtarget->hasFP16())) {
15348 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15349 }
15350 return Register();
15351}
15352
15353Register fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15354 if (RetVT.SimpleTy != MVT::v4f32)
15355 return Register();
15356 if ((Subtarget->hasAVX512())) {
15357 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15358 }
15359 return Register();
15360}
15361
15362Register fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15363 if (RetVT.SimpleTy != MVT::v2f64)
15364 return Register();
15365 if ((Subtarget->hasAVX512())) {
15366 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15367 }
15368 return Register();
15369}
15370
15371Register fastEmit_X86ISD_FDIVS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15372 switch (VT.SimpleTy) {
15373 case MVT::v8f16: return fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(RetVT, Op0, Op1);
15374 case MVT::v4f32: return fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(RetVT, Op0, Op1);
15375 case MVT::v2f64: return fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(RetVT, Op0, Op1);
15376 default: return Register();
15377 }
15378}
15379
15380// FastEmit functions for X86ISD::FGETEXPS.
15381
15382Register fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15383 if (RetVT.SimpleTy != MVT::v8f16)
15384 return Register();
15385 if ((Subtarget->hasFP16())) {
15386 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZr, RC: &X86::VR128XRegClass, Op0, Op1);
15387 }
15388 return Register();
15389}
15390
15391Register fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15392 if (RetVT.SimpleTy != MVT::v4f32)
15393 return Register();
15394 if ((Subtarget->hasAVX512())) {
15395 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZr, RC: &X86::VR128XRegClass, Op0, Op1);
15396 }
15397 return Register();
15398}
15399
15400Register fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15401 if (RetVT.SimpleTy != MVT::v2f64)
15402 return Register();
15403 if ((Subtarget->hasAVX512())) {
15404 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZr, RC: &X86::VR128XRegClass, Op0, Op1);
15405 }
15406 return Register();
15407}
15408
15409Register fastEmit_X86ISD_FGETEXPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15410 switch (VT.SimpleTy) {
15411 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(RetVT, Op0, Op1);
15412 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(RetVT, Op0, Op1);
15413 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(RetVT, Op0, Op1);
15414 default: return Register();
15415 }
15416}
15417
15418// FastEmit functions for X86ISD::FGETEXPS_SAE.
15419
15420Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15421 if (RetVT.SimpleTy != MVT::v8f16)
15422 return Register();
15423 if ((Subtarget->hasFP16())) {
15424 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZrb, RC: &X86::VR128XRegClass, Op0, Op1);
15425 }
15426 return Register();
15427}
15428
15429Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15430 if (RetVT.SimpleTy != MVT::v4f32)
15431 return Register();
15432 if ((Subtarget->hasAVX512())) {
15433 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZrb, RC: &X86::VR128XRegClass, Op0, Op1);
15434 }
15435 return Register();
15436}
15437
15438Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15439 if (RetVT.SimpleTy != MVT::v2f64)
15440 return Register();
15441 if ((Subtarget->hasAVX512())) {
15442 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZrb, RC: &X86::VR128XRegClass, Op0, Op1);
15443 }
15444 return Register();
15445}
15446
15447Register fastEmit_X86ISD_FGETEXPS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15448 switch (VT.SimpleTy) {
15449 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
15450 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
15451 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
15452 default: return Register();
15453 }
15454}
15455
15456// FastEmit functions for X86ISD::FHADD.
15457
15458Register fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15459 if (RetVT.SimpleTy != MVT::v4f32)
15460 return Register();
15461 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15462 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15463 }
15464 if ((Subtarget->hasAVX())) {
15465 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15466 }
15467 return Register();
15468}
15469
15470Register fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15471 if (RetVT.SimpleTy != MVT::v8f32)
15472 return Register();
15473 if ((Subtarget->hasAVX())) {
15474 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15475 }
15476 return Register();
15477}
15478
15479Register fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15480 if (RetVT.SimpleTy != MVT::v2f64)
15481 return Register();
15482 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15483 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15484 }
15485 if ((Subtarget->hasAVX())) {
15486 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15487 }
15488 return Register();
15489}
15490
15491Register fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15492 if (RetVT.SimpleTy != MVT::v4f64)
15493 return Register();
15494 if ((Subtarget->hasAVX())) {
15495 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15496 }
15497 return Register();
15498}
15499
15500Register fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15501 switch (VT.SimpleTy) {
15502 case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op1);
15503 case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op1);
15504 case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op1);
15505 case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op1);
15506 default: return Register();
15507 }
15508}
15509
15510// FastEmit functions for X86ISD::FHSUB.
15511
15512Register fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15513 if (RetVT.SimpleTy != MVT::v4f32)
15514 return Register();
15515 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15516 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15517 }
15518 if ((Subtarget->hasAVX())) {
15519 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15520 }
15521 return Register();
15522}
15523
15524Register fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15525 if (RetVT.SimpleTy != MVT::v8f32)
15526 return Register();
15527 if ((Subtarget->hasAVX())) {
15528 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15529 }
15530 return Register();
15531}
15532
15533Register fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15534 if (RetVT.SimpleTy != MVT::v2f64)
15535 return Register();
15536 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15537 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15538 }
15539 if ((Subtarget->hasAVX())) {
15540 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15541 }
15542 return Register();
15543}
15544
15545Register fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15546 if (RetVT.SimpleTy != MVT::v4f64)
15547 return Register();
15548 if ((Subtarget->hasAVX())) {
15549 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15550 }
15551 return Register();
15552}
15553
15554Register fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15555 switch (VT.SimpleTy) {
15556 case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
15557 case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
15558 case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
15559 case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
15560 default: return Register();
15561 }
15562}
15563
15564// FastEmit functions for X86ISD::FMAX.
15565
15566Register fastEmit_X86ISD_FMAX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
15567 if (RetVT.SimpleTy != MVT::f16)
15568 return Register();
15569 if ((Subtarget->hasFP16())) {
15570 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
15571 }
15572 return Register();
15573}
15574
15575Register fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
15576 if (RetVT.SimpleTy != MVT::f32)
15577 return Register();
15578 if ((Subtarget->hasAVX512())) {
15579 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
15580 }
15581 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15582 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15583 }
15584 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15585 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15586 }
15587 return Register();
15588}
15589
15590Register fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
15591 if (RetVT.SimpleTy != MVT::f64)
15592 return Register();
15593 if ((Subtarget->hasAVX512())) {
15594 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
15595 }
15596 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15597 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15598 }
15599 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15600 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15601 }
15602 return Register();
15603}
15604
15605Register fastEmit_X86ISD_FMAX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15606 if (RetVT.SimpleTy != MVT::v8f16)
15607 return Register();
15608 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15609 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15610 }
15611 return Register();
15612}
15613
15614Register fastEmit_X86ISD_FMAX_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
15615 if (RetVT.SimpleTy != MVT::v16f16)
15616 return Register();
15617 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15618 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15619 }
15620 return Register();
15621}
15622
15623Register fastEmit_X86ISD_FMAX_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
15624 if (RetVT.SimpleTy != MVT::v32f16)
15625 return Register();
15626 if ((Subtarget->hasFP16())) {
15627 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
15628 }
15629 return Register();
15630}
15631
15632Register fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15633 if (RetVT.SimpleTy != MVT::v4f32)
15634 return Register();
15635 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15636 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15637 }
15638 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15639 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15640 }
15641 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15642 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15643 }
15644 return Register();
15645}
15646
15647Register fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15648 if (RetVT.SimpleTy != MVT::v8f32)
15649 return Register();
15650 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15651 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15652 }
15653 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15654 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15655 }
15656 return Register();
15657}
15658
15659Register fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
15660 if (RetVT.SimpleTy != MVT::v16f32)
15661 return Register();
15662 if ((Subtarget->hasAVX512())) {
15663 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
15664 }
15665 return Register();
15666}
15667
15668Register fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15669 if (RetVT.SimpleTy != MVT::v2f64)
15670 return Register();
15671 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15672 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15673 }
15674 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15675 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15676 }
15677 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15678 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15679 }
15680 return Register();
15681}
15682
15683Register fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15684 if (RetVT.SimpleTy != MVT::v4f64)
15685 return Register();
15686 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15687 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15688 }
15689 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15690 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15691 }
15692 return Register();
15693}
15694
15695Register fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
15696 if (RetVT.SimpleTy != MVT::v8f64)
15697 return Register();
15698 if ((Subtarget->hasAVX512())) {
15699 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15700 }
15701 return Register();
15702}
15703
15704Register fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15705 switch (VT.SimpleTy) {
15706 case MVT::f16: return fastEmit_X86ISD_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
15707 case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
15708 case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
15709 case MVT::v8f16: return fastEmit_X86ISD_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
15710 case MVT::v16f16: return fastEmit_X86ISD_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
15711 case MVT::v32f16: return fastEmit_X86ISD_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
15712 case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
15713 case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
15714 case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
15715 case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
15716 case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
15717 case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
15718 default: return Register();
15719 }
15720}
15721
15722// FastEmit functions for X86ISD::FMAXC.
15723
15724Register fastEmit_X86ISD_FMAXC_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
15725 if (RetVT.SimpleTy != MVT::f16)
15726 return Register();
15727 if ((Subtarget->hasAVX512())) {
15728 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
15729 }
15730 return Register();
15731}
15732
15733Register fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
15734 if (RetVT.SimpleTy != MVT::f32)
15735 return Register();
15736 if ((Subtarget->hasAVX512())) {
15737 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
15738 }
15739 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15740 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15741 }
15742 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15743 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15744 }
15745 return Register();
15746}
15747
15748Register fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
15749 if (RetVT.SimpleTy != MVT::f64)
15750 return Register();
15751 if ((Subtarget->hasAVX512())) {
15752 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
15753 }
15754 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15755 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15756 }
15757 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15758 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15759 }
15760 return Register();
15761}
15762
15763Register fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15764 if (RetVT.SimpleTy != MVT::v8f16)
15765 return Register();
15766 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15767 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15768 }
15769 return Register();
15770}
15771
15772Register fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
15773 if (RetVT.SimpleTy != MVT::v16f16)
15774 return Register();
15775 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15776 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15777 }
15778 return Register();
15779}
15780
15781Register fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
15782 if (RetVT.SimpleTy != MVT::v32f16)
15783 return Register();
15784 if ((Subtarget->hasFP16())) {
15785 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
15786 }
15787 return Register();
15788}
15789
15790Register fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15791 if (RetVT.SimpleTy != MVT::v4f32)
15792 return Register();
15793 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15794 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15795 }
15796 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15797 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15798 }
15799 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15800 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15801 }
15802 return Register();
15803}
15804
15805Register fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15806 if (RetVT.SimpleTy != MVT::v8f32)
15807 return Register();
15808 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15809 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15810 }
15811 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15812 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15813 }
15814 return Register();
15815}
15816
15817Register fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
15818 if (RetVT.SimpleTy != MVT::v16f32)
15819 return Register();
15820 if ((Subtarget->hasAVX512())) {
15821 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
15822 }
15823 return Register();
15824}
15825
15826Register fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15827 if (RetVT.SimpleTy != MVT::v2f64)
15828 return Register();
15829 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15830 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15831 }
15832 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15833 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15834 }
15835 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15836 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15837 }
15838 return Register();
15839}
15840
15841Register fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15842 if (RetVT.SimpleTy != MVT::v4f64)
15843 return Register();
15844 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15845 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15846 }
15847 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15848 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15849 }
15850 return Register();
15851}
15852
15853Register fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
15854 if (RetVT.SimpleTy != MVT::v8f64)
15855 return Register();
15856 if ((Subtarget->hasAVX512())) {
15857 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15858 }
15859 return Register();
15860}
15861
15862Register fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15863 switch (VT.SimpleTy) {
15864 case MVT::f16: return fastEmit_X86ISD_FMAXC_MVT_f16_rr(RetVT, Op0, Op1);
15865 case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op1);
15866 case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op1);
15867 case MVT::v8f16: return fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(RetVT, Op0, Op1);
15868 case MVT::v16f16: return fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(RetVT, Op0, Op1);
15869 case MVT::v32f16: return fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(RetVT, Op0, Op1);
15870 case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op1);
15871 case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op1);
15872 case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op1);
15873 case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op1);
15874 case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op1);
15875 case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op1);
15876 default: return Register();
15877 }
15878}
15879
15880// FastEmit functions for X86ISD::FMAXS.
15881
15882Register fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15883 if (RetVT.SimpleTy != MVT::v8f16)
15884 return Register();
15885 if ((Subtarget->hasFP16())) {
15886 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15887 }
15888 return Register();
15889}
15890
15891Register fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15892 if (RetVT.SimpleTy != MVT::v4f32)
15893 return Register();
15894 if ((Subtarget->hasAVX512())) {
15895 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15896 }
15897 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15898 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15899 }
15900 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15901 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15902 }
15903 return Register();
15904}
15905
15906Register fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15907 if (RetVT.SimpleTy != MVT::v2f64)
15908 return Register();
15909 if ((Subtarget->hasAVX512())) {
15910 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15911 }
15912 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15913 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15914 }
15915 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15916 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15917 }
15918 return Register();
15919}
15920
15921Register fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15922 switch (VT.SimpleTy) {
15923 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(RetVT, Op0, Op1);
15924 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op1);
15925 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op1);
15926 default: return Register();
15927 }
15928}
15929
15930// FastEmit functions for X86ISD::FMAXS_SAE.
15931
15932Register fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15933 if (RetVT.SimpleTy != MVT::v8f16)
15934 return Register();
15935 if ((Subtarget->hasFP16())) {
15936 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15937 }
15938 return Register();
15939}
15940
15941Register fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15942 if (RetVT.SimpleTy != MVT::v4f32)
15943 return Register();
15944 if ((Subtarget->hasAVX512())) {
15945 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15946 }
15947 return Register();
15948}
15949
15950Register fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15951 if (RetVT.SimpleTy != MVT::v2f64)
15952 return Register();
15953 if ((Subtarget->hasAVX512())) {
15954 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15955 }
15956 return Register();
15957}
15958
15959Register fastEmit_X86ISD_FMAXS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15960 switch (VT.SimpleTy) {
15961 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
15962 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
15963 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
15964 default: return Register();
15965 }
15966}
15967
15968// FastEmit functions for X86ISD::FMAX_SAE.
15969
15970Register fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
15971 if (RetVT.SimpleTy != MVT::v32f16)
15972 return Register();
15973 if ((Subtarget->hasFP16())) {
15974 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
15975 }
15976 return Register();
15977}
15978
15979Register fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
15980 if (RetVT.SimpleTy != MVT::v16f32)
15981 return Register();
15982 if ((Subtarget->hasAVX512())) {
15983 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
15984 }
15985 return Register();
15986}
15987
15988Register fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
15989 if (RetVT.SimpleTy != MVT::v8f64)
15990 return Register();
15991 if ((Subtarget->hasAVX512())) {
15992 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
15993 }
15994 return Register();
15995}
15996
15997Register fastEmit_X86ISD_FMAX_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15998 switch (VT.SimpleTy) {
15999 case MVT::v32f16: return fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
16000 case MVT::v16f32: return fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
16001 case MVT::v8f64: return fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
16002 default: return Register();
16003 }
16004}
16005
16006// FastEmit functions for X86ISD::FMIN.
16007
16008Register fastEmit_X86ISD_FMIN_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16009 if (RetVT.SimpleTy != MVT::f16)
16010 return Register();
16011 if ((Subtarget->hasFP16())) {
16012 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16013 }
16014 return Register();
16015}
16016
16017Register fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16018 if (RetVT.SimpleTy != MVT::f32)
16019 return Register();
16020 if ((Subtarget->hasAVX512())) {
16021 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16022 }
16023 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16024 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16025 }
16026 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16027 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16028 }
16029 return Register();
16030}
16031
16032Register fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16033 if (RetVT.SimpleTy != MVT::f64)
16034 return Register();
16035 if ((Subtarget->hasAVX512())) {
16036 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16037 }
16038 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16039 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16040 }
16041 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16042 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16043 }
16044 return Register();
16045}
16046
16047Register fastEmit_X86ISD_FMIN_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16048 if (RetVT.SimpleTy != MVT::v8f16)
16049 return Register();
16050 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16051 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16052 }
16053 return Register();
16054}
16055
16056Register fastEmit_X86ISD_FMIN_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
16057 if (RetVT.SimpleTy != MVT::v16f16)
16058 return Register();
16059 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16060 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16061 }
16062 return Register();
16063}
16064
16065Register fastEmit_X86ISD_FMIN_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16066 if (RetVT.SimpleTy != MVT::v32f16)
16067 return Register();
16068 if ((Subtarget->hasFP16())) {
16069 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16070 }
16071 return Register();
16072}
16073
16074Register fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16075 if (RetVT.SimpleTy != MVT::v4f32)
16076 return Register();
16077 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16078 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16079 }
16080 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16081 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16082 }
16083 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16084 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16085 }
16086 return Register();
16087}
16088
16089Register fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16090 if (RetVT.SimpleTy != MVT::v8f32)
16091 return Register();
16092 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16093 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16094 }
16095 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16096 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16097 }
16098 return Register();
16099}
16100
16101Register fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16102 if (RetVT.SimpleTy != MVT::v16f32)
16103 return Register();
16104 if ((Subtarget->hasAVX512())) {
16105 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16106 }
16107 return Register();
16108}
16109
16110Register fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16111 if (RetVT.SimpleTy != MVT::v2f64)
16112 return Register();
16113 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16114 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16115 }
16116 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16117 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16118 }
16119 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16120 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16121 }
16122 return Register();
16123}
16124
16125Register fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16126 if (RetVT.SimpleTy != MVT::v4f64)
16127 return Register();
16128 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16129 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16130 }
16131 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16132 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16133 }
16134 return Register();
16135}
16136
16137Register fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16138 if (RetVT.SimpleTy != MVT::v8f64)
16139 return Register();
16140 if ((Subtarget->hasAVX512())) {
16141 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16142 }
16143 return Register();
16144}
16145
16146Register fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16147 switch (VT.SimpleTy) {
16148 case MVT::f16: return fastEmit_X86ISD_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
16149 case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
16150 case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
16151 case MVT::v8f16: return fastEmit_X86ISD_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
16152 case MVT::v16f16: return fastEmit_X86ISD_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
16153 case MVT::v32f16: return fastEmit_X86ISD_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
16154 case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
16155 case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
16156 case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
16157 case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
16158 case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
16159 case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
16160 default: return Register();
16161 }
16162}
16163
16164// FastEmit functions for X86ISD::FMINC.
16165
16166Register fastEmit_X86ISD_FMINC_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16167 if (RetVT.SimpleTy != MVT::f16)
16168 return Register();
16169 if ((Subtarget->hasAVX512())) {
16170 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16171 }
16172 return Register();
16173}
16174
16175Register fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16176 if (RetVT.SimpleTy != MVT::f32)
16177 return Register();
16178 if ((Subtarget->hasAVX512())) {
16179 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16180 }
16181 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16182 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16183 }
16184 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16185 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16186 }
16187 return Register();
16188}
16189
16190Register fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16191 if (RetVT.SimpleTy != MVT::f64)
16192 return Register();
16193 if ((Subtarget->hasAVX512())) {
16194 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16195 }
16196 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16197 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16198 }
16199 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16200 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16201 }
16202 return Register();
16203}
16204
16205Register fastEmit_X86ISD_FMINC_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16206 if (RetVT.SimpleTy != MVT::v8f16)
16207 return Register();
16208 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16209 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16210 }
16211 return Register();
16212}
16213
16214Register fastEmit_X86ISD_FMINC_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
16215 if (RetVT.SimpleTy != MVT::v16f16)
16216 return Register();
16217 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16218 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16219 }
16220 return Register();
16221}
16222
16223Register fastEmit_X86ISD_FMINC_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16224 if (RetVT.SimpleTy != MVT::v32f16)
16225 return Register();
16226 if ((Subtarget->hasFP16())) {
16227 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16228 }
16229 return Register();
16230}
16231
16232Register fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16233 if (RetVT.SimpleTy != MVT::v4f32)
16234 return Register();
16235 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16236 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16237 }
16238 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16239 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16240 }
16241 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16242 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16243 }
16244 return Register();
16245}
16246
16247Register fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16248 if (RetVT.SimpleTy != MVT::v8f32)
16249 return Register();
16250 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16251 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16252 }
16253 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16254 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16255 }
16256 return Register();
16257}
16258
16259Register fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16260 if (RetVT.SimpleTy != MVT::v16f32)
16261 return Register();
16262 if ((Subtarget->hasAVX512())) {
16263 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16264 }
16265 return Register();
16266}
16267
16268Register fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16269 if (RetVT.SimpleTy != MVT::v2f64)
16270 return Register();
16271 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16272 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16273 }
16274 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16275 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16276 }
16277 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16278 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16279 }
16280 return Register();
16281}
16282
16283Register fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16284 if (RetVT.SimpleTy != MVT::v4f64)
16285 return Register();
16286 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16287 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16288 }
16289 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16290 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16291 }
16292 return Register();
16293}
16294
16295Register fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16296 if (RetVT.SimpleTy != MVT::v8f64)
16297 return Register();
16298 if ((Subtarget->hasAVX512())) {
16299 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16300 }
16301 return Register();
16302}
16303
16304Register fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16305 switch (VT.SimpleTy) {
16306 case MVT::f16: return fastEmit_X86ISD_FMINC_MVT_f16_rr(RetVT, Op0, Op1);
16307 case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op1);
16308 case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op1);
16309 case MVT::v8f16: return fastEmit_X86ISD_FMINC_MVT_v8f16_rr(RetVT, Op0, Op1);
16310 case MVT::v16f16: return fastEmit_X86ISD_FMINC_MVT_v16f16_rr(RetVT, Op0, Op1);
16311 case MVT::v32f16: return fastEmit_X86ISD_FMINC_MVT_v32f16_rr(RetVT, Op0, Op1);
16312 case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op1);
16313 case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op1);
16314 case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op1);
16315 case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op1);
16316 case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op1);
16317 case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op1);
16318 default: return Register();
16319 }
16320}
16321
16322// FastEmit functions for X86ISD::FMINS.
16323
16324Register fastEmit_X86ISD_FMINS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16325 if (RetVT.SimpleTy != MVT::v8f16)
16326 return Register();
16327 if ((Subtarget->hasFP16())) {
16328 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16329 }
16330 return Register();
16331}
16332
16333Register fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16334 if (RetVT.SimpleTy != MVT::v4f32)
16335 return Register();
16336 if ((Subtarget->hasAVX512())) {
16337 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16338 }
16339 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16340 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16341 }
16342 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16343 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16344 }
16345 return Register();
16346}
16347
16348Register fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16349 if (RetVT.SimpleTy != MVT::v2f64)
16350 return Register();
16351 if ((Subtarget->hasAVX512())) {
16352 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16353 }
16354 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16355 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16356 }
16357 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16358 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16359 }
16360 return Register();
16361}
16362
16363Register fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16364 switch (VT.SimpleTy) {
16365 case MVT::v8f16: return fastEmit_X86ISD_FMINS_MVT_v8f16_rr(RetVT, Op0, Op1);
16366 case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op1);
16367 case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op1);
16368 default: return Register();
16369 }
16370}
16371
16372// FastEmit functions for X86ISD::FMINS_SAE.
16373
16374Register fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16375 if (RetVT.SimpleTy != MVT::v8f16)
16376 return Register();
16377 if ((Subtarget->hasFP16())) {
16378 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16379 }
16380 return Register();
16381}
16382
16383Register fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16384 if (RetVT.SimpleTy != MVT::v4f32)
16385 return Register();
16386 if ((Subtarget->hasAVX512())) {
16387 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16388 }
16389 return Register();
16390}
16391
16392Register fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16393 if (RetVT.SimpleTy != MVT::v2f64)
16394 return Register();
16395 if ((Subtarget->hasAVX512())) {
16396 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16397 }
16398 return Register();
16399}
16400
16401Register fastEmit_X86ISD_FMINS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16402 switch (VT.SimpleTy) {
16403 case MVT::v8f16: return fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
16404 case MVT::v4f32: return fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16405 case MVT::v2f64: return fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16406 default: return Register();
16407 }
16408}
16409
16410// FastEmit functions for X86ISD::FMIN_SAE.
16411
16412Register fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16413 if (RetVT.SimpleTy != MVT::v32f16)
16414 return Register();
16415 if ((Subtarget->hasFP16())) {
16416 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16417 }
16418 return Register();
16419}
16420
16421Register fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16422 if (RetVT.SimpleTy != MVT::v16f32)
16423 return Register();
16424 if ((Subtarget->hasAVX512())) {
16425 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16426 }
16427 return Register();
16428}
16429
16430Register fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16431 if (RetVT.SimpleTy != MVT::v8f64)
16432 return Register();
16433 if ((Subtarget->hasAVX512())) {
16434 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16435 }
16436 return Register();
16437}
16438
16439Register fastEmit_X86ISD_FMIN_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16440 switch (VT.SimpleTy) {
16441 case MVT::v32f16: return fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
16442 case MVT::v16f32: return fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
16443 case MVT::v8f64: return fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
16444 default: return Register();
16445 }
16446}
16447
16448// FastEmit functions for X86ISD::FMULS.
16449
16450Register fastEmit_X86ISD_FMULS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16451 if (RetVT.SimpleTy != MVT::v8f16)
16452 return Register();
16453 if ((Subtarget->hasFP16())) {
16454 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16455 }
16456 return Register();
16457}
16458
16459Register fastEmit_X86ISD_FMULS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16460 if (RetVT.SimpleTy != MVT::v4f32)
16461 return Register();
16462 if ((Subtarget->hasAVX512())) {
16463 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16464 }
16465 return Register();
16466}
16467
16468Register fastEmit_X86ISD_FMULS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16469 if (RetVT.SimpleTy != MVT::v2f64)
16470 return Register();
16471 if ((Subtarget->hasAVX512())) {
16472 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16473 }
16474 return Register();
16475}
16476
16477Register fastEmit_X86ISD_FMULS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16478 switch (VT.SimpleTy) {
16479 case MVT::v8f16: return fastEmit_X86ISD_FMULS_MVT_v8f16_rr(RetVT, Op0, Op1);
16480 case MVT::v4f32: return fastEmit_X86ISD_FMULS_MVT_v4f32_rr(RetVT, Op0, Op1);
16481 case MVT::v2f64: return fastEmit_X86ISD_FMULS_MVT_v2f64_rr(RetVT, Op0, Op1);
16482 default: return Register();
16483 }
16484}
16485
16486// FastEmit functions for X86ISD::FOR.
16487
16488Register fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
16489 if (RetVT.SimpleTy != MVT::f128)
16490 return Register();
16491 if ((Subtarget->hasVLX())) {
16492 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16493 }
16494 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16495 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16496 }
16497 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16498 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16499 }
16500 return Register();
16501}
16502
16503Register fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16504 if (RetVT.SimpleTy != MVT::v4f32)
16505 return Register();
16506 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16507}
16508
16509Register fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16510 switch (VT.SimpleTy) {
16511 case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op1);
16512 case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op1);
16513 default: return Register();
16514 }
16515}
16516
16517// FastEmit functions for X86ISD::FP80_ADD.
16518
16519Register fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
16520 if (RetVT.SimpleTy != MVT::f80)
16521 return Register();
16522 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
16523}
16524
16525Register fastEmit_X86ISD_FP80_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16526 switch (VT.SimpleTy) {
16527 case MVT::f80: return fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
16528 default: return Register();
16529 }
16530}
16531
16532// FastEmit functions for X86ISD::FSQRTS.
16533
16534Register fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16535 if (RetVT.SimpleTy != MVT::v8f16)
16536 return Register();
16537 if ((Subtarget->hasFP16())) {
16538 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSHZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16539 }
16540 return Register();
16541}
16542
16543Register fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16544 if (RetVT.SimpleTy != MVT::v4f32)
16545 return Register();
16546 if ((Subtarget->hasAVX512())) {
16547 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSSZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16548 }
16549 return Register();
16550}
16551
16552Register fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16553 if (RetVT.SimpleTy != MVT::v2f64)
16554 return Register();
16555 if ((Subtarget->hasAVX512())) {
16556 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSDZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16557 }
16558 return Register();
16559}
16560
16561Register fastEmit_X86ISD_FSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16562 switch (VT.SimpleTy) {
16563 case MVT::v8f16: return fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(RetVT, Op0, Op1);
16564 case MVT::v4f32: return fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
16565 case MVT::v2f64: return fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
16566 default: return Register();
16567 }
16568}
16569
16570// FastEmit functions for X86ISD::FSUBS.
16571
16572Register fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16573 if (RetVT.SimpleTy != MVT::v8f16)
16574 return Register();
16575 if ((Subtarget->hasFP16())) {
16576 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16577 }
16578 return Register();
16579}
16580
16581Register fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16582 if (RetVT.SimpleTy != MVT::v4f32)
16583 return Register();
16584 if ((Subtarget->hasAVX512())) {
16585 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16586 }
16587 return Register();
16588}
16589
16590Register fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16591 if (RetVT.SimpleTy != MVT::v2f64)
16592 return Register();
16593 if ((Subtarget->hasAVX512())) {
16594 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16595 }
16596 return Register();
16597}
16598
16599Register fastEmit_X86ISD_FSUBS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16600 switch (VT.SimpleTy) {
16601 case MVT::v8f16: return fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(RetVT, Op0, Op1);
16602 case MVT::v4f32: return fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(RetVT, Op0, Op1);
16603 case MVT::v2f64: return fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(RetVT, Op0, Op1);
16604 default: return Register();
16605 }
16606}
16607
16608// FastEmit functions for X86ISD::FXOR.
16609
16610Register fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
16611 if (RetVT.SimpleTy != MVT::f128)
16612 return Register();
16613 if ((Subtarget->hasVLX())) {
16614 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16615 }
16616 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16617 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16618 }
16619 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16620 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16621 }
16622 return Register();
16623}
16624
16625Register fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16626 if (RetVT.SimpleTy != MVT::v4f32)
16627 return Register();
16628 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16629}
16630
16631Register fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16632 switch (VT.SimpleTy) {
16633 case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op1);
16634 case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op1);
16635 default: return Register();
16636 }
16637}
16638
16639// FastEmit functions for X86ISD::GF2P8MULB.
16640
16641Register fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
16642 if (RetVT.SimpleTy != MVT::v16i8)
16643 return Register();
16644 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
16645 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16646 }
16647 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
16648 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
16649 }
16650 if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16651 return fastEmitInst_rr(MachineInstOpcode: X86::GF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
16652 }
16653 return Register();
16654}
16655
16656Register fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
16657 if (RetVT.SimpleTy != MVT::v32i8)
16658 return Register();
16659 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
16660 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16661 }
16662 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
16663 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBYrr, RC: &X86::VR256RegClass, Op0, Op1);
16664 }
16665 return Register();
16666}
16667
16668Register fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
16669 if (RetVT.SimpleTy != MVT::v64i8)
16670 return Register();
16671 if ((Subtarget->hasAVX512()) && (Subtarget->hasGFNI())) {
16672 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZrr, RC: &X86::VR512RegClass, Op0, Op1);
16673 }
16674 return Register();
16675}
16676
16677Register fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16678 switch (VT.SimpleTy) {
16679 case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op1);
16680 case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op1);
16681 case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op1);
16682 default: return Register();
16683 }
16684}
16685
16686// FastEmit functions for X86ISD::HADD.
16687
16688Register fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
16689 if (RetVT.SimpleTy != MVT::v8i16)
16690 return Register();
16691 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16692 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
16693 }
16694 if ((Subtarget->hasAVX())) {
16695 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
16696 }
16697 return Register();
16698}
16699
16700Register fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
16701 if (RetVT.SimpleTy != MVT::v16i16)
16702 return Register();
16703 if ((Subtarget->hasAVX2())) {
16704 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16705 }
16706 return Register();
16707}
16708
16709Register fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
16710 if (RetVT.SimpleTy != MVT::v4i32)
16711 return Register();
16712 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16713 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
16714 }
16715 if ((Subtarget->hasAVX())) {
16716 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
16717 }
16718 return Register();
16719}
16720
16721Register fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
16722 if (RetVT.SimpleTy != MVT::v8i32)
16723 return Register();
16724 if ((Subtarget->hasAVX2())) {
16725 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16726 }
16727 return Register();
16728}
16729
16730Register fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16731 switch (VT.SimpleTy) {
16732 case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op1);
16733 case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op1);
16734 case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op1);
16735 case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op1);
16736 default: return Register();
16737 }
16738}
16739
16740// FastEmit functions for X86ISD::HSUB.
16741
16742Register fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
16743 if (RetVT.SimpleTy != MVT::v8i16)
16744 return Register();
16745 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16746 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
16747 }
16748 if ((Subtarget->hasAVX())) {
16749 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
16750 }
16751 return Register();
16752}
16753
16754Register fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
16755 if (RetVT.SimpleTy != MVT::v16i16)
16756 return Register();
16757 if ((Subtarget->hasAVX2())) {
16758 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16759 }
16760 return Register();
16761}
16762
16763Register fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
16764 if (RetVT.SimpleTy != MVT::v4i32)
16765 return Register();
16766 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16767 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
16768 }
16769 if ((Subtarget->hasAVX())) {
16770 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
16771 }
16772 return Register();
16773}
16774
16775Register fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
16776 if (RetVT.SimpleTy != MVT::v8i32)
16777 return Register();
16778 if ((Subtarget->hasAVX2())) {
16779 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16780 }
16781 return Register();
16782}
16783
16784Register fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16785 switch (VT.SimpleTy) {
16786 case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op1);
16787 case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op1);
16788 case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op1);
16789 case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op1);
16790 default: return Register();
16791 }
16792}
16793
16794// FastEmit functions for X86ISD::KADD.
16795
16796Register fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
16797 if (RetVT.SimpleTy != MVT::v8i1)
16798 return Register();
16799 if ((Subtarget->hasDQI())) {
16800 return fastEmitInst_rr(MachineInstOpcode: X86::KADDBkk, RC: &X86::VK8RegClass, Op0, Op1);
16801 }
16802 return Register();
16803}
16804
16805Register fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
16806 if (RetVT.SimpleTy != MVT::v16i1)
16807 return Register();
16808 if ((Subtarget->hasDQI())) {
16809 return fastEmitInst_rr(MachineInstOpcode: X86::KADDWkk, RC: &X86::VK16RegClass, Op0, Op1);
16810 }
16811 return Register();
16812}
16813
16814Register fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
16815 if (RetVT.SimpleTy != MVT::v32i1)
16816 return Register();
16817 if ((Subtarget->hasBWI())) {
16818 return fastEmitInst_rr(MachineInstOpcode: X86::KADDDkk, RC: &X86::VK32RegClass, Op0, Op1);
16819 }
16820 return Register();
16821}
16822
16823Register fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
16824 if (RetVT.SimpleTy != MVT::v64i1)
16825 return Register();
16826 if ((Subtarget->hasBWI())) {
16827 return fastEmitInst_rr(MachineInstOpcode: X86::KADDQkk, RC: &X86::VK64RegClass, Op0, Op1);
16828 }
16829 return Register();
16830}
16831
16832Register fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16833 switch (VT.SimpleTy) {
16834 case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op1);
16835 case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op1);
16836 case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op1);
16837 case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op1);
16838 default: return Register();
16839 }
16840}
16841
16842// FastEmit functions for X86ISD::KORTEST.
16843
16844Register fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
16845 if (RetVT.SimpleTy != MVT::i32)
16846 return Register();
16847 if ((Subtarget->hasDQI())) {
16848 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTBkk, RC: &X86::VK8RegClass, Op0, Op1);
16849 }
16850 return Register();
16851}
16852
16853Register fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
16854 if (RetVT.SimpleTy != MVT::i32)
16855 return Register();
16856 if ((Subtarget->hasAVX512())) {
16857 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTWkk, RC: &X86::VK16RegClass, Op0, Op1);
16858 }
16859 return Register();
16860}
16861
16862Register fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
16863 if (RetVT.SimpleTy != MVT::i32)
16864 return Register();
16865 if ((Subtarget->hasBWI())) {
16866 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTDkk, RC: &X86::VK32RegClass, Op0, Op1);
16867 }
16868 return Register();
16869}
16870
16871Register fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
16872 if (RetVT.SimpleTy != MVT::i32)
16873 return Register();
16874 if ((Subtarget->hasBWI())) {
16875 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTQkk, RC: &X86::VK64RegClass, Op0, Op1);
16876 }
16877 return Register();
16878}
16879
16880Register fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16881 switch (VT.SimpleTy) {
16882 case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
16883 case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
16884 case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
16885 case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
16886 default: return Register();
16887 }
16888}
16889
16890// FastEmit functions for X86ISD::KTEST.
16891
16892Register fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
16893 if (RetVT.SimpleTy != MVT::i32)
16894 return Register();
16895 if ((Subtarget->hasDQI())) {
16896 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTBkk, RC: &X86::VK8RegClass, Op0, Op1);
16897 }
16898 return Register();
16899}
16900
16901Register fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
16902 if (RetVT.SimpleTy != MVT::i32)
16903 return Register();
16904 if ((Subtarget->hasDQI())) {
16905 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTWkk, RC: &X86::VK16RegClass, Op0, Op1);
16906 }
16907 return Register();
16908}
16909
16910Register fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
16911 if (RetVT.SimpleTy != MVT::i32)
16912 return Register();
16913 if ((Subtarget->hasBWI())) {
16914 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTDkk, RC: &X86::VK32RegClass, Op0, Op1);
16915 }
16916 return Register();
16917}
16918
16919Register fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
16920 if (RetVT.SimpleTy != MVT::i32)
16921 return Register();
16922 if ((Subtarget->hasBWI())) {
16923 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTQkk, RC: &X86::VK64RegClass, Op0, Op1);
16924 }
16925 return Register();
16926}
16927
16928Register fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16929 switch (VT.SimpleTy) {
16930 case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
16931 case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
16932 case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
16933 case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
16934 default: return Register();
16935 }
16936}
16937
16938// FastEmit functions for X86ISD::MOVHLPS.
16939
16940Register fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16941 if (RetVT.SimpleTy != MVT::v4f32)
16942 return Register();
16943 if ((Subtarget->hasAVX512())) {
16944 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16945 }
16946 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16947 return fastEmitInst_rr(MachineInstOpcode: X86::MOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16948 }
16949 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16950 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16951 }
16952 return Register();
16953}
16954
16955Register fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16956 switch (VT.SimpleTy) {
16957 case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op1);
16958 default: return Register();
16959 }
16960}
16961
16962// FastEmit functions for X86ISD::MOVLHPS.
16963
16964Register fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16965 if (RetVT.SimpleTy != MVT::v4f32)
16966 return Register();
16967 if ((Subtarget->hasAVX512())) {
16968 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16969 }
16970 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16971 return fastEmitInst_rr(MachineInstOpcode: X86::MOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16972 }
16973 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16974 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16975 }
16976 return Register();
16977}
16978
16979Register fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16980 switch (VT.SimpleTy) {
16981 case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op1);
16982 default: return Register();
16983 }
16984}
16985
16986// FastEmit functions for X86ISD::MOVSD.
16987
16988Register fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16989 if (RetVT.SimpleTy != MVT::v2f64)
16990 return Register();
16991 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
16992 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
16993 }
16994 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16995 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
16996 }
16997 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16998 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
16999 }
17000 return Register();
17001}
17002
17003Register fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17004 switch (VT.SimpleTy) {
17005 case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op1);
17006 default: return Register();
17007 }
17008}
17009
17010// FastEmit functions for X86ISD::MOVSH.
17011
17012Register fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17013 if (RetVT.SimpleTy != MVT::v8f16)
17014 return Register();
17015 if ((Subtarget->hasFP16())) {
17016 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17017 }
17018 return Register();
17019}
17020
17021Register fastEmit_X86ISD_MOVSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17022 switch (VT.SimpleTy) {
17023 case MVT::v8f16: return fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(RetVT, Op0, Op1);
17024 default: return Register();
17025 }
17026}
17027
17028// FastEmit functions for X86ISD::MOVSS.
17029
17030Register fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17031 if (RetVT.SimpleTy != MVT::v4f32)
17032 return Register();
17033 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
17034 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17035 }
17036 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17037 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
17038 }
17039 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17040 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
17041 }
17042 return Register();
17043}
17044
17045Register fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17046 switch (VT.SimpleTy) {
17047 case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op1);
17048 default: return Register();
17049 }
17050}
17051
17052// FastEmit functions for X86ISD::MULHRS.
17053
17054Register fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17055 if (RetVT.SimpleTy != MVT::v8i16)
17056 return Register();
17057 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17058 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17059 }
17060 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17061 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17062 }
17063 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17064 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17065 }
17066 return Register();
17067}
17068
17069Register fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17070 if (RetVT.SimpleTy != MVT::v16i16)
17071 return Register();
17072 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17073 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17074 }
17075 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17076 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17077 }
17078 return Register();
17079}
17080
17081Register fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17082 if (RetVT.SimpleTy != MVT::v32i16)
17083 return Register();
17084 if ((Subtarget->hasBWI())) {
17085 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17086 }
17087 return Register();
17088}
17089
17090Register fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17091 switch (VT.SimpleTy) {
17092 case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op1);
17093 case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op1);
17094 case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op1);
17095 default: return Register();
17096 }
17097}
17098
17099// FastEmit functions for X86ISD::MULTISHIFT.
17100
17101Register fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17102 if (RetVT.SimpleTy != MVT::v16i8)
17103 return Register();
17104 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17105 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17106 }
17107 return Register();
17108}
17109
17110Register fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17111 if (RetVT.SimpleTy != MVT::v32i8)
17112 return Register();
17113 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17114 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17115 }
17116 return Register();
17117}
17118
17119Register fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17120 if (RetVT.SimpleTy != MVT::v64i8)
17121 return Register();
17122 if ((Subtarget->hasVBMI())) {
17123 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17124 }
17125 return Register();
17126}
17127
17128Register fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17129 switch (VT.SimpleTy) {
17130 case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op1);
17131 case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op1);
17132 case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op1);
17133 default: return Register();
17134 }
17135}
17136
17137// FastEmit functions for X86ISD::PACKSS.
17138
17139Register fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17140 if (RetVT.SimpleTy != MVT::v16i8)
17141 return Register();
17142 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17143 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17144 }
17145 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17146 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17147 }
17148 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17149 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17150 }
17151 return Register();
17152}
17153
17154Register fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17155 if (RetVT.SimpleTy != MVT::v32i8)
17156 return Register();
17157 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17158 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17159 }
17160 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17161 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17162 }
17163 return Register();
17164}
17165
17166Register fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17167 if (RetVT.SimpleTy != MVT::v64i8)
17168 return Register();
17169 if ((Subtarget->hasBWI())) {
17170 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17171 }
17172 return Register();
17173}
17174
17175Register fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17176 if (RetVT.SimpleTy != MVT::v8i16)
17177 return Register();
17178 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17179 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17180 }
17181 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17182 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17183 }
17184 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17185 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17186 }
17187 return Register();
17188}
17189
17190Register fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17191 if (RetVT.SimpleTy != MVT::v16i16)
17192 return Register();
17193 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17194 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17195 }
17196 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17197 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17198 }
17199 return Register();
17200}
17201
17202Register fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
17203 if (RetVT.SimpleTy != MVT::v32i16)
17204 return Register();
17205 if ((Subtarget->hasBWI())) {
17206 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17207 }
17208 return Register();
17209}
17210
17211Register fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17212 switch (VT.SimpleTy) {
17213 case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op1);
17214 case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op1);
17215 case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op1);
17216 case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op1);
17217 case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op1);
17218 case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op1);
17219 default: return Register();
17220 }
17221}
17222
17223// FastEmit functions for X86ISD::PACKUS.
17224
17225Register fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17226 if (RetVT.SimpleTy != MVT::v16i8)
17227 return Register();
17228 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17229 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17230 }
17231 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17232 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17233 }
17234 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17235 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17236 }
17237 return Register();
17238}
17239
17240Register fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17241 if (RetVT.SimpleTy != MVT::v32i8)
17242 return Register();
17243 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17244 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17245 }
17246 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17247 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17248 }
17249 return Register();
17250}
17251
17252Register fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17253 if (RetVT.SimpleTy != MVT::v64i8)
17254 return Register();
17255 if ((Subtarget->hasBWI())) {
17256 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17257 }
17258 return Register();
17259}
17260
17261Register fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17262 if (RetVT.SimpleTy != MVT::v8i16)
17263 return Register();
17264 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17265 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17266 }
17267 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17268 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17269 }
17270 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17271 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17272 }
17273 return Register();
17274}
17275
17276Register fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17277 if (RetVT.SimpleTy != MVT::v16i16)
17278 return Register();
17279 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17280 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17281 }
17282 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17283 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17284 }
17285 return Register();
17286}
17287
17288Register fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
17289 if (RetVT.SimpleTy != MVT::v32i16)
17290 return Register();
17291 if ((Subtarget->hasBWI())) {
17292 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17293 }
17294 return Register();
17295}
17296
17297Register fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17298 switch (VT.SimpleTy) {
17299 case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op1);
17300 case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op1);
17301 case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op1);
17302 case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op1);
17303 case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op1);
17304 case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op1);
17305 default: return Register();
17306 }
17307}
17308
17309// FastEmit functions for X86ISD::PCMPEQ.
17310
17311Register fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17312 if (RetVT.SimpleTy != MVT::v16i8)
17313 return Register();
17314 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17315 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
17316 }
17317 if ((Subtarget->hasAVX()) && (true)) {
17318 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
17319 }
17320 return Register();
17321}
17322
17323Register fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17324 if (RetVT.SimpleTy != MVT::v32i8)
17325 return Register();
17326 if ((Subtarget->hasAVX2()) && (true)) {
17327 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17328 }
17329 return Register();
17330}
17331
17332Register fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17333 if (RetVT.SimpleTy != MVT::v8i16)
17334 return Register();
17335 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17336 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
17337 }
17338 if ((Subtarget->hasAVX()) && (true)) {
17339 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
17340 }
17341 return Register();
17342}
17343
17344Register fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17345 if (RetVT.SimpleTy != MVT::v16i16)
17346 return Register();
17347 if ((Subtarget->hasAVX2()) && (true)) {
17348 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17349 }
17350 return Register();
17351}
17352
17353Register fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17354 if (RetVT.SimpleTy != MVT::v4i32)
17355 return Register();
17356 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17357 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
17358 }
17359 if ((Subtarget->hasAVX()) && (true)) {
17360 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
17361 }
17362 return Register();
17363}
17364
17365Register fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17366 if (RetVT.SimpleTy != MVT::v8i32)
17367 return Register();
17368 if ((Subtarget->hasAVX2()) && (true)) {
17369 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17370 }
17371 return Register();
17372}
17373
17374Register fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17375 if (RetVT.SimpleTy != MVT::v2i64)
17376 return Register();
17377 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17378 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
17379 }
17380 if ((Subtarget->hasAVX())) {
17381 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
17382 }
17383 return Register();
17384}
17385
17386Register fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17387 if (RetVT.SimpleTy != MVT::v4i64)
17388 return Register();
17389 if ((Subtarget->hasAVX2())) {
17390 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17391 }
17392 return Register();
17393}
17394
17395Register fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17396 switch (VT.SimpleTy) {
17397 case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op1);
17398 case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op1);
17399 case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op1);
17400 case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op1);
17401 case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op1);
17402 case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op1);
17403 case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17404 case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17405 default: return Register();
17406 }
17407}
17408
17409// FastEmit functions for X86ISD::PCMPGT.
17410
17411Register fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17412 if (RetVT.SimpleTy != MVT::v16i8)
17413 return Register();
17414 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17415 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
17416 }
17417 if ((Subtarget->hasAVX()) && (true)) {
17418 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
17419 }
17420 return Register();
17421}
17422
17423Register fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17424 if (RetVT.SimpleTy != MVT::v32i8)
17425 return Register();
17426 if ((Subtarget->hasAVX2()) && (true)) {
17427 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17428 }
17429 return Register();
17430}
17431
17432Register fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17433 if (RetVT.SimpleTy != MVT::v8i16)
17434 return Register();
17435 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17436 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
17437 }
17438 if ((Subtarget->hasAVX()) && (true)) {
17439 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
17440 }
17441 return Register();
17442}
17443
17444Register fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17445 if (RetVT.SimpleTy != MVT::v16i16)
17446 return Register();
17447 if ((Subtarget->hasAVX2()) && (true)) {
17448 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17449 }
17450 return Register();
17451}
17452
17453Register fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17454 if (RetVT.SimpleTy != MVT::v4i32)
17455 return Register();
17456 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17457 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
17458 }
17459 if ((Subtarget->hasAVX()) && (true)) {
17460 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
17461 }
17462 return Register();
17463}
17464
17465Register fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17466 if (RetVT.SimpleTy != MVT::v8i32)
17467 return Register();
17468 if ((Subtarget->hasAVX2()) && (true)) {
17469 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17470 }
17471 return Register();
17472}
17473
17474Register fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17475 if (RetVT.SimpleTy != MVT::v2i64)
17476 return Register();
17477 if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) {
17478 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
17479 }
17480 if ((Subtarget->hasAVX())) {
17481 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
17482 }
17483 return Register();
17484}
17485
17486Register fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17487 if (RetVT.SimpleTy != MVT::v4i64)
17488 return Register();
17489 if ((Subtarget->hasAVX2())) {
17490 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17491 }
17492 return Register();
17493}
17494
17495Register fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17496 switch (VT.SimpleTy) {
17497 case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op1);
17498 case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op1);
17499 case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op1);
17500 case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op1);
17501 case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op1);
17502 case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op1);
17503 case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op1);
17504 case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op1);
17505 default: return Register();
17506 }
17507}
17508
17509// FastEmit functions for X86ISD::PDEP.
17510
17511Register fastEmit_X86ISD_PDEP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
17512 if (RetVT.SimpleTy != MVT::i32)
17513 return Register();
17514 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
17515 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
17516 }
17517 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
17518 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr, RC: &X86::GR32RegClass, Op0, Op1);
17519 }
17520 return Register();
17521}
17522
17523Register fastEmit_X86ISD_PDEP_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
17524 if (RetVT.SimpleTy != MVT::i64)
17525 return Register();
17526 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
17527 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
17528 }
17529 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
17530 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr, RC: &X86::GR64RegClass, Op0, Op1);
17531 }
17532 return Register();
17533}
17534
17535Register fastEmit_X86ISD_PDEP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17536 switch (VT.SimpleTy) {
17537 case MVT::i32: return fastEmit_X86ISD_PDEP_MVT_i32_rr(RetVT, Op0, Op1);
17538 case MVT::i64: return fastEmit_X86ISD_PDEP_MVT_i64_rr(RetVT, Op0, Op1);
17539 default: return Register();
17540 }
17541}
17542
17543// FastEmit functions for X86ISD::PEXT.
17544
17545Register fastEmit_X86ISD_PEXT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
17546 if (RetVT.SimpleTy != MVT::i32)
17547 return Register();
17548 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
17549 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
17550 }
17551 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
17552 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr, RC: &X86::GR32RegClass, Op0, Op1);
17553 }
17554 return Register();
17555}
17556
17557Register fastEmit_X86ISD_PEXT_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
17558 if (RetVT.SimpleTy != MVT::i64)
17559 return Register();
17560 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
17561 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
17562 }
17563 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
17564 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr, RC: &X86::GR64RegClass, Op0, Op1);
17565 }
17566 return Register();
17567}
17568
17569Register fastEmit_X86ISD_PEXT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17570 switch (VT.SimpleTy) {
17571 case MVT::i32: return fastEmit_X86ISD_PEXT_MVT_i32_rr(RetVT, Op0, Op1);
17572 case MVT::i64: return fastEmit_X86ISD_PEXT_MVT_i64_rr(RetVT, Op0, Op1);
17573 default: return Register();
17574 }
17575}
17576
17577// FastEmit functions for X86ISD::PMULDQ.
17578
17579Register fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17580 if (RetVT.SimpleTy != MVT::v2i64)
17581 return Register();
17582 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17583 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17584 }
17585 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17586 return fastEmitInst_rr(MachineInstOpcode: X86::PMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17587 }
17588 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17589 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17590 }
17591 return Register();
17592}
17593
17594Register fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17595 if (RetVT.SimpleTy != MVT::v4i64)
17596 return Register();
17597 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17598 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17599 }
17600 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17601 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17602 }
17603 return Register();
17604}
17605
17606Register fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
17607 if (RetVT.SimpleTy != MVT::v8i64)
17608 return Register();
17609 if ((Subtarget->hasAVX512())) {
17610 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17611 }
17612 return Register();
17613}
17614
17615Register fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17616 switch (VT.SimpleTy) {
17617 case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17618 case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17619 case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
17620 default: return Register();
17621 }
17622}
17623
17624// FastEmit functions for X86ISD::PMULUDQ.
17625
17626Register fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17627 if (RetVT.SimpleTy != MVT::v2i64)
17628 return Register();
17629 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17630 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17631 }
17632 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17633 return fastEmitInst_rr(MachineInstOpcode: X86::PMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17634 }
17635 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17636 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17637 }
17638 return Register();
17639}
17640
17641Register fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17642 if (RetVT.SimpleTy != MVT::v4i64)
17643 return Register();
17644 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17645 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17646 }
17647 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17648 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17649 }
17650 return Register();
17651}
17652
17653Register fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
17654 if (RetVT.SimpleTy != MVT::v8i64)
17655 return Register();
17656 if ((Subtarget->hasAVX512())) {
17657 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17658 }
17659 return Register();
17660}
17661
17662Register fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17663 switch (VT.SimpleTy) {
17664 case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17665 case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17666 case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
17667 default: return Register();
17668 }
17669}
17670
17671// FastEmit functions for X86ISD::PSADBW.
17672
17673Register fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17674 if (RetVT.SimpleTy != MVT::v2i64)
17675 return Register();
17676 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17677 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17678 }
17679 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17680 return fastEmitInst_rr(MachineInstOpcode: X86::PSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17681 }
17682 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17683 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17684 }
17685 return Register();
17686}
17687
17688Register fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17689 if (RetVT.SimpleTy != MVT::v4i64)
17690 return Register();
17691 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17692 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17693 }
17694 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17695 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17696 }
17697 return Register();
17698}
17699
17700Register fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17701 if (RetVT.SimpleTy != MVT::v8i64)
17702 return Register();
17703 if ((Subtarget->hasBWI())) {
17704 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17705 }
17706 return Register();
17707}
17708
17709Register fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17710 switch (VT.SimpleTy) {
17711 case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op1);
17712 case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op1);
17713 case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op1);
17714 default: return Register();
17715 }
17716}
17717
17718// FastEmit functions for X86ISD::PSHUFB.
17719
17720Register fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17721 if (RetVT.SimpleTy != MVT::v16i8)
17722 return Register();
17723 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17724 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17725 }
17726 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17727 return fastEmitInst_rr(MachineInstOpcode: X86::PSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
17728 }
17729 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17730 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
17731 }
17732 return Register();
17733}
17734
17735Register fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17736 if (RetVT.SimpleTy != MVT::v32i8)
17737 return Register();
17738 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17739 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17740 }
17741 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17742 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17743 }
17744 return Register();
17745}
17746
17747Register fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17748 if (RetVT.SimpleTy != MVT::v64i8)
17749 return Register();
17750 if ((Subtarget->hasBWI())) {
17751 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17752 }
17753 return Register();
17754}
17755
17756Register fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17757 switch (VT.SimpleTy) {
17758 case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op1);
17759 case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op1);
17760 case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op1);
17761 default: return Register();
17762 }
17763}
17764
17765// FastEmit functions for X86ISD::PTEST.
17766
17767Register fastEmit_X86ISD_PTEST_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17768 if (RetVT.SimpleTy != MVT::i32)
17769 return Register();
17770 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17771 return fastEmitInst_rr(MachineInstOpcode: X86::PTESTrr, RC: &X86::VR128RegClass, Op0, Op1);
17772 }
17773 if ((Subtarget->hasAVX())) {
17774 return fastEmitInst_rr(MachineInstOpcode: X86::VPTESTrr, RC: &X86::VR128RegClass, Op0, Op1);
17775 }
17776 return Register();
17777}
17778
17779Register fastEmit_X86ISD_PTEST_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17780 if (RetVT.SimpleTy != MVT::i32)
17781 return Register();
17782 if ((Subtarget->hasAVX())) {
17783 return fastEmitInst_rr(MachineInstOpcode: X86::VPTESTYrr, RC: &X86::VR256RegClass, Op0, Op1);
17784 }
17785 return Register();
17786}
17787
17788Register fastEmit_X86ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17789 switch (VT.SimpleTy) {
17790 case MVT::v2i64: return fastEmit_X86ISD_PTEST_MVT_v2i64_rr(RetVT, Op0, Op1);
17791 case MVT::v4i64: return fastEmit_X86ISD_PTEST_MVT_v4i64_rr(RetVT, Op0, Op1);
17792 default: return Register();
17793 }
17794}
17795
17796// FastEmit functions for X86ISD::RCP14S.
17797
17798Register fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17799 if (RetVT.SimpleTy != MVT::v8f16)
17800 return Register();
17801 if ((Subtarget->hasFP16())) {
17802 return fastEmitInst_rr(MachineInstOpcode: X86::VRCPSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17803 }
17804 return Register();
17805}
17806
17807Register fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17808 if (RetVT.SimpleTy != MVT::v4f32)
17809 return Register();
17810 if ((Subtarget->hasAVX512())) {
17811 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17812 }
17813 return Register();
17814}
17815
17816Register fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17817 if (RetVT.SimpleTy != MVT::v2f64)
17818 return Register();
17819 if ((Subtarget->hasAVX512())) {
17820 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17821 }
17822 return Register();
17823}
17824
17825Register fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17826 switch (VT.SimpleTy) {
17827 case MVT::v8f16: return fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(RetVT, Op0, Op1);
17828 case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op1);
17829 case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op1);
17830 default: return Register();
17831 }
17832}
17833
17834// FastEmit functions for X86ISD::RSQRT14S.
17835
17836Register fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17837 if (RetVT.SimpleTy != MVT::v8f16)
17838 return Register();
17839 if ((Subtarget->hasFP16())) {
17840 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRTSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17841 }
17842 return Register();
17843}
17844
17845Register fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17846 if (RetVT.SimpleTy != MVT::v4f32)
17847 return Register();
17848 if ((Subtarget->hasAVX512())) {
17849 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17850 }
17851 return Register();
17852}
17853
17854Register fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17855 if (RetVT.SimpleTy != MVT::v2f64)
17856 return Register();
17857 if ((Subtarget->hasAVX512())) {
17858 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17859 }
17860 return Register();
17861}
17862
17863Register fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17864 switch (VT.SimpleTy) {
17865 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(RetVT, Op0, Op1);
17866 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op1);
17867 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op1);
17868 default: return Register();
17869 }
17870}
17871
17872// FastEmit functions for X86ISD::SCALEF.
17873
17874Register fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17875 if (RetVT.SimpleTy != MVT::v8f16)
17876 return Register();
17877 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17878 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17879 }
17880 return Register();
17881}
17882
17883Register fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
17884 if (RetVT.SimpleTy != MVT::v16f16)
17885 return Register();
17886 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17887 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17888 }
17889 return Register();
17890}
17891
17892Register fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
17893 if (RetVT.SimpleTy != MVT::v32f16)
17894 return Register();
17895 if ((Subtarget->hasFP16())) {
17896 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
17897 }
17898 return Register();
17899}
17900
17901Register fastEmit_X86ISD_SCALEF_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
17902 if (RetVT.SimpleTy != MVT::v8bf16)
17903 return Register();
17904 if ((Subtarget->hasAVX10_2())) {
17905 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17906 }
17907 return Register();
17908}
17909
17910Register fastEmit_X86ISD_SCALEF_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
17911 if (RetVT.SimpleTy != MVT::v16bf16)
17912 return Register();
17913 if ((Subtarget->hasAVX10_2())) {
17914 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17915 }
17916 return Register();
17917}
17918
17919Register fastEmit_X86ISD_SCALEF_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
17920 if (RetVT.SimpleTy != MVT::v32bf16)
17921 return Register();
17922 if ((Subtarget->hasAVX10_2_512())) {
17923 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
17924 }
17925 return Register();
17926}
17927
17928Register fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17929 if (RetVT.SimpleTy != MVT::v4f32)
17930 return Register();
17931 if ((Subtarget->hasVLX())) {
17932 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17933 }
17934 return Register();
17935}
17936
17937Register fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
17938 if (RetVT.SimpleTy != MVT::v8f32)
17939 return Register();
17940 if ((Subtarget->hasVLX())) {
17941 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17942 }
17943 return Register();
17944}
17945
17946Register fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
17947 if (RetVT.SimpleTy != MVT::v16f32)
17948 return Register();
17949 if ((Subtarget->hasAVX512())) {
17950 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
17951 }
17952 return Register();
17953}
17954
17955Register fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17956 if (RetVT.SimpleTy != MVT::v2f64)
17957 return Register();
17958 if ((Subtarget->hasVLX())) {
17959 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17960 }
17961 return Register();
17962}
17963
17964Register fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
17965 if (RetVT.SimpleTy != MVT::v4f64)
17966 return Register();
17967 if ((Subtarget->hasVLX())) {
17968 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17969 }
17970 return Register();
17971}
17972
17973Register fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
17974 if (RetVT.SimpleTy != MVT::v8f64)
17975 return Register();
17976 if ((Subtarget->hasAVX512())) {
17977 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
17978 }
17979 return Register();
17980}
17981
17982Register fastEmit_X86ISD_SCALEF_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17983 switch (VT.SimpleTy) {
17984 case MVT::v8f16: return fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(RetVT, Op0, Op1);
17985 case MVT::v16f16: return fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(RetVT, Op0, Op1);
17986 case MVT::v32f16: return fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(RetVT, Op0, Op1);
17987 case MVT::v8bf16: return fastEmit_X86ISD_SCALEF_MVT_v8bf16_rr(RetVT, Op0, Op1);
17988 case MVT::v16bf16: return fastEmit_X86ISD_SCALEF_MVT_v16bf16_rr(RetVT, Op0, Op1);
17989 case MVT::v32bf16: return fastEmit_X86ISD_SCALEF_MVT_v32bf16_rr(RetVT, Op0, Op1);
17990 case MVT::v4f32: return fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(RetVT, Op0, Op1);
17991 case MVT::v8f32: return fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(RetVT, Op0, Op1);
17992 case MVT::v16f32: return fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(RetVT, Op0, Op1);
17993 case MVT::v2f64: return fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(RetVT, Op0, Op1);
17994 case MVT::v4f64: return fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(RetVT, Op0, Op1);
17995 case MVT::v8f64: return fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(RetVT, Op0, Op1);
17996 default: return Register();
17997 }
17998}
17999
18000// FastEmit functions for X86ISD::SCALEFS.
18001
18002Register fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18003 if (RetVT.SimpleTy != MVT::v8f16)
18004 return Register();
18005 if ((Subtarget->hasFP16())) {
18006 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18007 }
18008 return Register();
18009}
18010
18011Register fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18012 if (RetVT.SimpleTy != MVT::v4f32)
18013 return Register();
18014 if ((Subtarget->hasAVX512())) {
18015 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18016 }
18017 return Register();
18018}
18019
18020Register fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18021 if (RetVT.SimpleTy != MVT::v2f64)
18022 return Register();
18023 if ((Subtarget->hasAVX512())) {
18024 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18025 }
18026 return Register();
18027}
18028
18029Register fastEmit_X86ISD_SCALEFS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18030 switch (VT.SimpleTy) {
18031 case MVT::v8f16: return fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(RetVT, Op0, Op1);
18032 case MVT::v4f32: return fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(RetVT, Op0, Op1);
18033 case MVT::v2f64: return fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(RetVT, Op0, Op1);
18034 default: return Register();
18035 }
18036}
18037
18038// FastEmit functions for X86ISD::STRICT_FCMP.
18039
18040Register fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18041 if (RetVT.SimpleTy != MVT::i32)
18042 return Register();
18043 if ((Subtarget->hasFP16())) {
18044 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18045 }
18046 return Register();
18047}
18048
18049Register fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18050 if (RetVT.SimpleTy != MVT::i32)
18051 return Register();
18052 if ((Subtarget->hasAVX512())) {
18053 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18054 }
18055 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18056 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18057 }
18058 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18059 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18060 }
18061 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
18062 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
18063 }
18064 return Register();
18065}
18066
18067Register fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18068 if (RetVT.SimpleTy != MVT::i32)
18069 return Register();
18070 if ((Subtarget->hasAVX512())) {
18071 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18072 }
18073 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18074 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18075 }
18076 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18077 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18078 }
18079 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
18080 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
18081 }
18082 return Register();
18083}
18084
18085Register fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18086 if (RetVT.SimpleTy != MVT::i32)
18087 return Register();
18088 if ((Subtarget->canUseCMOV())) {
18089 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
18090 }
18091 return Register();
18092}
18093
18094Register fastEmit_X86ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18095 switch (VT.SimpleTy) {
18096 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
18097 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
18098 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
18099 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
18100 default: return Register();
18101 }
18102}
18103
18104// FastEmit functions for X86ISD::STRICT_FCMPS.
18105
18106Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18107 if (RetVT.SimpleTy != MVT::i32)
18108 return Register();
18109 if ((Subtarget->hasFP16())) {
18110 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18111 }
18112 return Register();
18113}
18114
18115Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18116 if (RetVT.SimpleTy != MVT::i32)
18117 return Register();
18118 if ((Subtarget->hasAVX512())) {
18119 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18120 }
18121 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18122 return fastEmitInst_rr(MachineInstOpcode: X86::COMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18123 }
18124 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18125 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18126 }
18127 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
18128 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
18129 }
18130 return Register();
18131}
18132
18133Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18134 if (RetVT.SimpleTy != MVT::i32)
18135 return Register();
18136 if ((Subtarget->hasAVX512())) {
18137 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18138 }
18139 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18140 return fastEmitInst_rr(MachineInstOpcode: X86::COMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18141 }
18142 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18143 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18144 }
18145 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
18146 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
18147 }
18148 return Register();
18149}
18150
18151Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18152 if (RetVT.SimpleTy != MVT::i32)
18153 return Register();
18154 if ((Subtarget->canUseCMOV())) {
18155 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
18156 }
18157 return Register();
18158}
18159
18160Register fastEmit_X86ISD_STRICT_FCMPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18161 switch (VT.SimpleTy) {
18162 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(RetVT, Op0, Op1);
18163 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(RetVT, Op0, Op1);
18164 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(RetVT, Op0, Op1);
18165 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(RetVT, Op0, Op1);
18166 default: return Register();
18167 }
18168}
18169
18170// FastEmit functions for X86ISD::STRICT_FMAX.
18171
18172Register fastEmit_X86ISD_STRICT_FMAX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18173 if (RetVT.SimpleTy != MVT::f16)
18174 return Register();
18175 if ((Subtarget->hasFP16())) {
18176 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18177 }
18178 return Register();
18179}
18180
18181Register fastEmit_X86ISD_STRICT_FMAX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18182 if (RetVT.SimpleTy != MVT::f32)
18183 return Register();
18184 if ((Subtarget->hasAVX512())) {
18185 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18186 }
18187 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18188 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18189 }
18190 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18191 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18192 }
18193 return Register();
18194}
18195
18196Register fastEmit_X86ISD_STRICT_FMAX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18197 if (RetVT.SimpleTy != MVT::f64)
18198 return Register();
18199 if ((Subtarget->hasAVX512())) {
18200 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18201 }
18202 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18203 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18204 }
18205 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18206 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18207 }
18208 return Register();
18209}
18210
18211Register fastEmit_X86ISD_STRICT_FMAX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18212 if (RetVT.SimpleTy != MVT::v8f16)
18213 return Register();
18214 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18215 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18216 }
18217 return Register();
18218}
18219
18220Register fastEmit_X86ISD_STRICT_FMAX_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18221 if (RetVT.SimpleTy != MVT::v16f16)
18222 return Register();
18223 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18224 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18225 }
18226 return Register();
18227}
18228
18229Register fastEmit_X86ISD_STRICT_FMAX_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18230 if (RetVT.SimpleTy != MVT::v32f16)
18231 return Register();
18232 if ((Subtarget->hasFP16())) {
18233 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
18234 }
18235 return Register();
18236}
18237
18238Register fastEmit_X86ISD_STRICT_FMAX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18239 if (RetVT.SimpleTy != MVT::v4f32)
18240 return Register();
18241 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18242 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18243 }
18244 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18245 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18246 }
18247 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18248 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18249 }
18250 return Register();
18251}
18252
18253Register fastEmit_X86ISD_STRICT_FMAX_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18254 if (RetVT.SimpleTy != MVT::v8f32)
18255 return Register();
18256 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18257 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18258 }
18259 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18260 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18261 }
18262 return Register();
18263}
18264
18265Register fastEmit_X86ISD_STRICT_FMAX_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18266 if (RetVT.SimpleTy != MVT::v16f32)
18267 return Register();
18268 if ((Subtarget->hasAVX512())) {
18269 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18270 }
18271 return Register();
18272}
18273
18274Register fastEmit_X86ISD_STRICT_FMAX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18275 if (RetVT.SimpleTy != MVT::v2f64)
18276 return Register();
18277 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18278 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18279 }
18280 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18281 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18282 }
18283 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18284 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18285 }
18286 return Register();
18287}
18288
18289Register fastEmit_X86ISD_STRICT_FMAX_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18290 if (RetVT.SimpleTy != MVT::v4f64)
18291 return Register();
18292 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18293 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18294 }
18295 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18296 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18297 }
18298 return Register();
18299}
18300
18301Register fastEmit_X86ISD_STRICT_FMAX_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18302 if (RetVT.SimpleTy != MVT::v8f64)
18303 return Register();
18304 if ((Subtarget->hasAVX512())) {
18305 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18306 }
18307 return Register();
18308}
18309
18310Register fastEmit_X86ISD_STRICT_FMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18311 switch (VT.SimpleTy) {
18312 case MVT::f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
18313 case MVT::f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
18314 case MVT::f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
18315 case MVT::v8f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
18316 case MVT::v16f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
18317 case MVT::v32f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
18318 case MVT::v4f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
18319 case MVT::v8f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
18320 case MVT::v16f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
18321 case MVT::v2f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
18322 case MVT::v4f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
18323 case MVT::v8f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
18324 default: return Register();
18325 }
18326}
18327
18328// FastEmit functions for X86ISD::STRICT_FMIN.
18329
18330Register fastEmit_X86ISD_STRICT_FMIN_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18331 if (RetVT.SimpleTy != MVT::f16)
18332 return Register();
18333 if ((Subtarget->hasFP16())) {
18334 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18335 }
18336 return Register();
18337}
18338
18339Register fastEmit_X86ISD_STRICT_FMIN_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18340 if (RetVT.SimpleTy != MVT::f32)
18341 return Register();
18342 if ((Subtarget->hasAVX512())) {
18343 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18344 }
18345 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18346 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18347 }
18348 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18349 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18350 }
18351 return Register();
18352}
18353
18354Register fastEmit_X86ISD_STRICT_FMIN_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18355 if (RetVT.SimpleTy != MVT::f64)
18356 return Register();
18357 if ((Subtarget->hasAVX512())) {
18358 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18359 }
18360 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18361 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18362 }
18363 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18364 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18365 }
18366 return Register();
18367}
18368
18369Register fastEmit_X86ISD_STRICT_FMIN_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18370 if (RetVT.SimpleTy != MVT::v8f16)
18371 return Register();
18372 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18373 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18374 }
18375 return Register();
18376}
18377
18378Register fastEmit_X86ISD_STRICT_FMIN_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18379 if (RetVT.SimpleTy != MVT::v16f16)
18380 return Register();
18381 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18382 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18383 }
18384 return Register();
18385}
18386
18387Register fastEmit_X86ISD_STRICT_FMIN_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18388 if (RetVT.SimpleTy != MVT::v32f16)
18389 return Register();
18390 if ((Subtarget->hasFP16())) {
18391 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
18392 }
18393 return Register();
18394}
18395
18396Register fastEmit_X86ISD_STRICT_FMIN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18397 if (RetVT.SimpleTy != MVT::v4f32)
18398 return Register();
18399 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18400 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18401 }
18402 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18403 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18404 }
18405 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18406 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18407 }
18408 return Register();
18409}
18410
18411Register fastEmit_X86ISD_STRICT_FMIN_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18412 if (RetVT.SimpleTy != MVT::v8f32)
18413 return Register();
18414 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18415 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18416 }
18417 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18418 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18419 }
18420 return Register();
18421}
18422
18423Register fastEmit_X86ISD_STRICT_FMIN_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18424 if (RetVT.SimpleTy != MVT::v16f32)
18425 return Register();
18426 if ((Subtarget->hasAVX512())) {
18427 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18428 }
18429 return Register();
18430}
18431
18432Register fastEmit_X86ISD_STRICT_FMIN_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18433 if (RetVT.SimpleTy != MVT::v2f64)
18434 return Register();
18435 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18436 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18437 }
18438 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18439 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18440 }
18441 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18442 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18443 }
18444 return Register();
18445}
18446
18447Register fastEmit_X86ISD_STRICT_FMIN_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18448 if (RetVT.SimpleTy != MVT::v4f64)
18449 return Register();
18450 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18451 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18452 }
18453 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18454 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18455 }
18456 return Register();
18457}
18458
18459Register fastEmit_X86ISD_STRICT_FMIN_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18460 if (RetVT.SimpleTy != MVT::v8f64)
18461 return Register();
18462 if ((Subtarget->hasAVX512())) {
18463 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18464 }
18465 return Register();
18466}
18467
18468Register fastEmit_X86ISD_STRICT_FMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18469 switch (VT.SimpleTy) {
18470 case MVT::f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
18471 case MVT::f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
18472 case MVT::f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
18473 case MVT::v8f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
18474 case MVT::v16f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
18475 case MVT::v32f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
18476 case MVT::v4f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
18477 case MVT::v8f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
18478 case MVT::v16f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
18479 case MVT::v2f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
18480 case MVT::v4f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
18481 case MVT::v8f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
18482 default: return Register();
18483 }
18484}
18485
18486// FastEmit functions for X86ISD::STRICT_FP80_ADD.
18487
18488Register fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18489 if (RetVT.SimpleTy != MVT::f80)
18490 return Register();
18491 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
18492}
18493
18494Register fastEmit_X86ISD_STRICT_FP80_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18495 switch (VT.SimpleTy) {
18496 case MVT::f80: return fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
18497 default: return Register();
18498 }
18499}
18500
18501// FastEmit functions for X86ISD::TESTP.
18502
18503Register fastEmit_X86ISD_TESTP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18504 if (RetVT.SimpleTy != MVT::i32)
18505 return Register();
18506 if ((Subtarget->hasAVX())) {
18507 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18508 }
18509 return Register();
18510}
18511
18512Register fastEmit_X86ISD_TESTP_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18513 if (RetVT.SimpleTy != MVT::i32)
18514 return Register();
18515 if ((Subtarget->hasAVX())) {
18516 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18517 }
18518 return Register();
18519}
18520
18521Register fastEmit_X86ISD_TESTP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18522 if (RetVT.SimpleTy != MVT::i32)
18523 return Register();
18524 if ((Subtarget->hasAVX())) {
18525 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18526 }
18527 return Register();
18528}
18529
18530Register fastEmit_X86ISD_TESTP_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18531 if (RetVT.SimpleTy != MVT::i32)
18532 return Register();
18533 if ((Subtarget->hasAVX())) {
18534 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18535 }
18536 return Register();
18537}
18538
18539Register fastEmit_X86ISD_TESTP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18540 switch (VT.SimpleTy) {
18541 case MVT::v4f32: return fastEmit_X86ISD_TESTP_MVT_v4f32_rr(RetVT, Op0, Op1);
18542 case MVT::v8f32: return fastEmit_X86ISD_TESTP_MVT_v8f32_rr(RetVT, Op0, Op1);
18543 case MVT::v2f64: return fastEmit_X86ISD_TESTP_MVT_v2f64_rr(RetVT, Op0, Op1);
18544 case MVT::v4f64: return fastEmit_X86ISD_TESTP_MVT_v4f64_rr(RetVT, Op0, Op1);
18545 default: return Register();
18546 }
18547}
18548
18549// FastEmit functions for X86ISD::UCOMI.
18550
18551Register fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18552 if (RetVT.SimpleTy != MVT::i32)
18553 return Register();
18554 if ((Subtarget->hasFP16())) {
18555 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18556 }
18557 return Register();
18558}
18559
18560Register fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18561 if (RetVT.SimpleTy != MVT::i32)
18562 return Register();
18563 if ((Subtarget->hasAVX512())) {
18564 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18565 }
18566 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18567 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18568 }
18569 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18570 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18571 }
18572 return Register();
18573}
18574
18575Register fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18576 if (RetVT.SimpleTy != MVT::i32)
18577 return Register();
18578 if ((Subtarget->hasAVX512())) {
18579 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18580 }
18581 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18582 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18583 }
18584 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18585 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18586 }
18587 return Register();
18588}
18589
18590Register fastEmit_X86ISD_UCOMI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18591 switch (VT.SimpleTy) {
18592 case MVT::v8f16: return fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(RetVT, Op0, Op1);
18593 case MVT::v4f32: return fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(RetVT, Op0, Op1);
18594 case MVT::v2f64: return fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(RetVT, Op0, Op1);
18595 default: return Register();
18596 }
18597}
18598
18599// FastEmit functions for X86ISD::UCOMX.
18600
18601Register fastEmit_X86ISD_UCOMX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18602 if (RetVT.SimpleTy != MVT::i32)
18603 return Register();
18604 if ((Subtarget->hasAVX10_2())) {
18605 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18606 }
18607 return Register();
18608}
18609
18610Register fastEmit_X86ISD_UCOMX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18611 if (RetVT.SimpleTy != MVT::i32)
18612 return Register();
18613 if ((Subtarget->hasAVX10_2())) {
18614 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18615 }
18616 return Register();
18617}
18618
18619Register fastEmit_X86ISD_UCOMX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18620 if (RetVT.SimpleTy != MVT::i32)
18621 return Register();
18622 if ((Subtarget->hasAVX10_2())) {
18623 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18624 }
18625 return Register();
18626}
18627
18628Register fastEmit_X86ISD_UCOMX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18629 if (RetVT.SimpleTy != MVT::i32)
18630 return Register();
18631 if ((Subtarget->hasAVX10_2())) {
18632 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18633 }
18634 return Register();
18635}
18636
18637Register fastEmit_X86ISD_UCOMX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18638 if (RetVT.SimpleTy != MVT::i32)
18639 return Register();
18640 if ((Subtarget->hasAVX10_2())) {
18641 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18642 }
18643 return Register();
18644}
18645
18646Register fastEmit_X86ISD_UCOMX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18647 if (RetVT.SimpleTy != MVT::i32)
18648 return Register();
18649 if ((Subtarget->hasAVX10_2())) {
18650 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18651 }
18652 return Register();
18653}
18654
18655Register fastEmit_X86ISD_UCOMX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18656 switch (VT.SimpleTy) {
18657 case MVT::f16: return fastEmit_X86ISD_UCOMX_MVT_f16_rr(RetVT, Op0, Op1);
18658 case MVT::f32: return fastEmit_X86ISD_UCOMX_MVT_f32_rr(RetVT, Op0, Op1);
18659 case MVT::f64: return fastEmit_X86ISD_UCOMX_MVT_f64_rr(RetVT, Op0, Op1);
18660 case MVT::v8f16: return fastEmit_X86ISD_UCOMX_MVT_v8f16_rr(RetVT, Op0, Op1);
18661 case MVT::v4f32: return fastEmit_X86ISD_UCOMX_MVT_v4f32_rr(RetVT, Op0, Op1);
18662 case MVT::v2f64: return fastEmit_X86ISD_UCOMX_MVT_v2f64_rr(RetVT, Op0, Op1);
18663 default: return Register();
18664 }
18665}
18666
18667// FastEmit functions for X86ISD::UNPCKH.
18668
18669Register fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18670 if (RetVT.SimpleTy != MVT::v16i8)
18671 return Register();
18672 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18673 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18674 }
18675 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18676 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18677 }
18678 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18679 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18680 }
18681 return Register();
18682}
18683
18684Register fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18685 if (RetVT.SimpleTy != MVT::v32i8)
18686 return Register();
18687 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18688 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18689 }
18690 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18691 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
18692 }
18693 return Register();
18694}
18695
18696Register fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
18697 if (RetVT.SimpleTy != MVT::v64i8)
18698 return Register();
18699 if ((Subtarget->hasBWI())) {
18700 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
18701 }
18702 return Register();
18703}
18704
18705Register fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
18706 if (RetVT.SimpleTy != MVT::v8i16)
18707 return Register();
18708 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18709 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18710 }
18711 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18712 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18713 }
18714 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18715 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18716 }
18717 return Register();
18718}
18719
18720Register fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
18721 if (RetVT.SimpleTy != MVT::v16i16)
18722 return Register();
18723 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18724 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18725 }
18726 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18727 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18728 }
18729 return Register();
18730}
18731
18732Register fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
18733 if (RetVT.SimpleTy != MVT::v32i16)
18734 return Register();
18735 if ((Subtarget->hasBWI())) {
18736 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18737 }
18738 return Register();
18739}
18740
18741Register fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
18742 if (RetVT.SimpleTy != MVT::v4i32)
18743 return Register();
18744 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18745 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18746 }
18747 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18748 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18749 }
18750 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18751 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18752 }
18753 return Register();
18754}
18755
18756Register fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
18757 if (RetVT.SimpleTy != MVT::v8i32)
18758 return Register();
18759 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18760 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18761 }
18762 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18763 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18764 }
18765 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18766 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18767 }
18768 return Register();
18769}
18770
18771Register fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
18772 if (RetVT.SimpleTy != MVT::v16i32)
18773 return Register();
18774 if ((Subtarget->hasAVX512())) {
18775 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18776 }
18777 return Register();
18778}
18779
18780Register fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
18781 if (RetVT.SimpleTy != MVT::v2i64)
18782 return Register();
18783 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18784 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18785 }
18786 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18787 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18788 }
18789 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18790 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18791 }
18792 return Register();
18793}
18794
18795Register fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
18796 if (RetVT.SimpleTy != MVT::v4i64)
18797 return Register();
18798 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18799 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18800 }
18801 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18802 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18803 }
18804 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18805 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18806 }
18807 return Register();
18808}
18809
18810Register fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
18811 if (RetVT.SimpleTy != MVT::v8i64)
18812 return Register();
18813 if ((Subtarget->hasAVX512())) {
18814 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18815 }
18816 return Register();
18817}
18818
18819Register fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18820 if (RetVT.SimpleTy != MVT::v4f32)
18821 return Register();
18822 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18823 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18824 }
18825 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18826 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18827 }
18828 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18829 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18830 }
18831 return Register();
18832}
18833
18834Register fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18835 if (RetVT.SimpleTy != MVT::v8f32)
18836 return Register();
18837 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18838 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18839 }
18840 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18841 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18842 }
18843 return Register();
18844}
18845
18846Register fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18847 if (RetVT.SimpleTy != MVT::v16f32)
18848 return Register();
18849 if ((Subtarget->hasAVX512())) {
18850 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18851 }
18852 return Register();
18853}
18854
18855Register fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18856 if (RetVT.SimpleTy != MVT::v2f64)
18857 return Register();
18858 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18859 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18860 }
18861 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18862 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18863 }
18864 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18865 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18866 }
18867 return Register();
18868}
18869
18870Register fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18871 if (RetVT.SimpleTy != MVT::v4f64)
18872 return Register();
18873 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18874 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18875 }
18876 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18877 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18878 }
18879 return Register();
18880}
18881
18882Register fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18883 if (RetVT.SimpleTy != MVT::v8f64)
18884 return Register();
18885 if ((Subtarget->hasAVX512())) {
18886 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18887 }
18888 return Register();
18889}
18890
18891Register fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18892 switch (VT.SimpleTy) {
18893 case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op1);
18894 case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op1);
18895 case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op1);
18896 case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op1);
18897 case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op1);
18898 case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op1);
18899 case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op1);
18900 case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op1);
18901 case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op1);
18902 case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op1);
18903 case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op1);
18904 case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op1);
18905 case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op1);
18906 case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op1);
18907 case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op1);
18908 case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op1);
18909 case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op1);
18910 case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op1);
18911 default: return Register();
18912 }
18913}
18914
18915// FastEmit functions for X86ISD::UNPCKL.
18916
18917Register fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18918 if (RetVT.SimpleTy != MVT::v16i8)
18919 return Register();
18920 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18921 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18922 }
18923 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18924 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18925 }
18926 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18927 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18928 }
18929 return Register();
18930}
18931
18932Register fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18933 if (RetVT.SimpleTy != MVT::v32i8)
18934 return Register();
18935 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18936 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18937 }
18938 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18939 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
18940 }
18941 return Register();
18942}
18943
18944Register fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
18945 if (RetVT.SimpleTy != MVT::v64i8)
18946 return Register();
18947 if ((Subtarget->hasBWI())) {
18948 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
18949 }
18950 return Register();
18951}
18952
18953Register fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
18954 if (RetVT.SimpleTy != MVT::v8i16)
18955 return Register();
18956 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18957 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18958 }
18959 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18960 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18961 }
18962 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18963 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18964 }
18965 return Register();
18966}
18967
18968Register fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
18969 if (RetVT.SimpleTy != MVT::v16i16)
18970 return Register();
18971 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18972 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18973 }
18974 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18975 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18976 }
18977 return Register();
18978}
18979
18980Register fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
18981 if (RetVT.SimpleTy != MVT::v32i16)
18982 return Register();
18983 if ((Subtarget->hasBWI())) {
18984 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18985 }
18986 return Register();
18987}
18988
18989Register fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
18990 if (RetVT.SimpleTy != MVT::v4i32)
18991 return Register();
18992 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18993 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18994 }
18995 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18996 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18997 }
18998 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18999 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19000 }
19001 return Register();
19002}
19003
19004Register fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19005 if (RetVT.SimpleTy != MVT::v8i32)
19006 return Register();
19007 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
19008 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
19009 }
19010 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19011 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19012 }
19013 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19014 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
19015 }
19016 return Register();
19017}
19018
19019Register fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19020 if (RetVT.SimpleTy != MVT::v16i32)
19021 return Register();
19022 if ((Subtarget->hasAVX512())) {
19023 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19024 }
19025 return Register();
19026}
19027
19028Register fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19029 if (RetVT.SimpleTy != MVT::v2i64)
19030 return Register();
19031 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19032 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19033 }
19034 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19035 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19036 }
19037 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19038 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19039 }
19040 return Register();
19041}
19042
19043Register fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19044 if (RetVT.SimpleTy != MVT::v4i64)
19045 return Register();
19046 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
19047 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19048 }
19049 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19050 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19051 }
19052 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19053 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
19054 }
19055 return Register();
19056}
19057
19058Register fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19059 if (RetVT.SimpleTy != MVT::v8i64)
19060 return Register();
19061 if ((Subtarget->hasAVX512())) {
19062 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19063 }
19064 return Register();
19065}
19066
19067Register fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19068 if (RetVT.SimpleTy != MVT::v4f32)
19069 return Register();
19070 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19071 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19072 }
19073 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
19074 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
19075 }
19076 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19077 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
19078 }
19079 return Register();
19080}
19081
19082Register fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19083 if (RetVT.SimpleTy != MVT::v8f32)
19084 return Register();
19085 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19086 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19087 }
19088 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19089 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
19090 }
19091 return Register();
19092}
19093
19094Register fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19095 if (RetVT.SimpleTy != MVT::v16f32)
19096 return Register();
19097 if ((Subtarget->hasAVX512())) {
19098 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
19099 }
19100 return Register();
19101}
19102
19103Register fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
19104 if (RetVT.SimpleTy != MVT::v2f64)
19105 return Register();
19106 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19107 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19108 }
19109 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19110 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
19111 }
19112 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19113 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
19114 }
19115 return Register();
19116}
19117
19118Register fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
19119 if (RetVT.SimpleTy != MVT::v4f64)
19120 return Register();
19121 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19122 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19123 }
19124 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19125 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19126 }
19127 return Register();
19128}
19129
19130Register fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
19131 if (RetVT.SimpleTy != MVT::v8f64)
19132 return Register();
19133 if ((Subtarget->hasAVX512())) {
19134 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19135 }
19136 return Register();
19137}
19138
19139Register fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19140 switch (VT.SimpleTy) {
19141 case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op1);
19142 case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op1);
19143 case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op1);
19144 case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op1);
19145 case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op1);
19146 case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op1);
19147 case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op1);
19148 case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op1);
19149 case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op1);
19150 case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op1);
19151 case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op1);
19152 case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op1);
19153 case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op1);
19154 case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op1);
19155 case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op1);
19156 case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op1);
19157 case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op1);
19158 case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op1);
19159 default: return Register();
19160 }
19161}
19162
19163// FastEmit functions for X86ISD::VCVT2PH2BF8.
19164
19165Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19166 if (RetVT.SimpleTy != MVT::v16i8)
19167 return Register();
19168 if ((Subtarget->hasAVX10_2())) {
19169 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19170 }
19171 return Register();
19172}
19173
19174Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19175 if (RetVT.SimpleTy != MVT::v32i8)
19176 return Register();
19177 if ((Subtarget->hasAVX10_2())) {
19178 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19179 }
19180 return Register();
19181}
19182
19183Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19184 if (RetVT.SimpleTy != MVT::v64i8)
19185 return Register();
19186 if ((Subtarget->hasAVX10_2_512())) {
19187 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19188 }
19189 return Register();
19190}
19191
19192Register fastEmit_X86ISD_VCVT2PH2BF8_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19193 switch (VT.SimpleTy) {
19194 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v8f16_rr(RetVT, Op0, Op1);
19195 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v16f16_rr(RetVT, Op0, Op1);
19196 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v32f16_rr(RetVT, Op0, Op1);
19197 default: return Register();
19198 }
19199}
19200
19201// FastEmit functions for X86ISD::VCVT2PH2BF8S.
19202
19203Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19204 if (RetVT.SimpleTy != MVT::v16i8)
19205 return Register();
19206 if ((Subtarget->hasAVX10_2())) {
19207 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19208 }
19209 return Register();
19210}
19211
19212Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19213 if (RetVT.SimpleTy != MVT::v32i8)
19214 return Register();
19215 if ((Subtarget->hasAVX10_2())) {
19216 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19217 }
19218 return Register();
19219}
19220
19221Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19222 if (RetVT.SimpleTy != MVT::v64i8)
19223 return Register();
19224 if ((Subtarget->hasAVX10_2_512())) {
19225 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZrr, RC: &X86::VR512RegClass, Op0, Op1);
19226 }
19227 return Register();
19228}
19229
19230Register fastEmit_X86ISD_VCVT2PH2BF8S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19231 switch (VT.SimpleTy) {
19232 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v8f16_rr(RetVT, Op0, Op1);
19233 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v16f16_rr(RetVT, Op0, Op1);
19234 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v32f16_rr(RetVT, Op0, Op1);
19235 default: return Register();
19236 }
19237}
19238
19239// FastEmit functions for X86ISD::VCVT2PH2HF8.
19240
19241Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19242 if (RetVT.SimpleTy != MVT::v16i8)
19243 return Register();
19244 if ((Subtarget->hasAVX10_2())) {
19245 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19246 }
19247 return Register();
19248}
19249
19250Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19251 if (RetVT.SimpleTy != MVT::v32i8)
19252 return Register();
19253 if ((Subtarget->hasAVX10_2())) {
19254 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19255 }
19256 return Register();
19257}
19258
19259Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19260 if (RetVT.SimpleTy != MVT::v64i8)
19261 return Register();
19262 if ((Subtarget->hasAVX10_2_512())) {
19263 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19264 }
19265 return Register();
19266}
19267
19268Register fastEmit_X86ISD_VCVT2PH2HF8_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19269 switch (VT.SimpleTy) {
19270 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v8f16_rr(RetVT, Op0, Op1);
19271 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v16f16_rr(RetVT, Op0, Op1);
19272 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v32f16_rr(RetVT, Op0, Op1);
19273 default: return Register();
19274 }
19275}
19276
19277// FastEmit functions for X86ISD::VCVT2PH2HF8S.
19278
19279Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19280 if (RetVT.SimpleTy != MVT::v16i8)
19281 return Register();
19282 if ((Subtarget->hasAVX10_2())) {
19283 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19284 }
19285 return Register();
19286}
19287
19288Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19289 if (RetVT.SimpleTy != MVT::v32i8)
19290 return Register();
19291 if ((Subtarget->hasAVX10_2())) {
19292 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19293 }
19294 return Register();
19295}
19296
19297Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19298 if (RetVT.SimpleTy != MVT::v64i8)
19299 return Register();
19300 if ((Subtarget->hasAVX10_2_512())) {
19301 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZrr, RC: &X86::VR512RegClass, Op0, Op1);
19302 }
19303 return Register();
19304}
19305
19306Register fastEmit_X86ISD_VCVT2PH2HF8S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19307 switch (VT.SimpleTy) {
19308 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v8f16_rr(RetVT, Op0, Op1);
19309 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v16f16_rr(RetVT, Op0, Op1);
19310 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v32f16_rr(RetVT, Op0, Op1);
19311 default: return Register();
19312 }
19313}
19314
19315// FastEmit functions for X86ISD::VFCMULC.
19316
19317Register fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19318 if (RetVT.SimpleTy != MVT::v4f32)
19319 return Register();
19320 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19321 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19322 }
19323 return Register();
19324}
19325
19326Register fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19327 if (RetVT.SimpleTy != MVT::v8f32)
19328 return Register();
19329 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19330 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19331 }
19332 return Register();
19333}
19334
19335Register fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19336 if (RetVT.SimpleTy != MVT::v16f32)
19337 return Register();
19338 if ((Subtarget->hasFP16())) {
19339 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
19340 }
19341 return Register();
19342}
19343
19344Register fastEmit_X86ISD_VFCMULC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19345 switch (VT.SimpleTy) {
19346 case MVT::v4f32: return fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
19347 case MVT::v8f32: return fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
19348 case MVT::v16f32: return fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
19349 default: return Register();
19350 }
19351}
19352
19353// FastEmit functions for X86ISD::VFCMULCSH.
19354
19355Register fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19356 if (RetVT.SimpleTy != MVT::v4f32)
19357 return Register();
19358 if ((Subtarget->hasFP16())) {
19359 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
19360 }
19361 return Register();
19362}
19363
19364Register fastEmit_X86ISD_VFCMULCSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19365 switch (VT.SimpleTy) {
19366 case MVT::v4f32: return fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
19367 default: return Register();
19368 }
19369}
19370
19371// FastEmit functions for X86ISD::VFMULC.
19372
19373Register fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19374 if (RetVT.SimpleTy != MVT::v4f32)
19375 return Register();
19376 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19377 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19378 }
19379 return Register();
19380}
19381
19382Register fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19383 if (RetVT.SimpleTy != MVT::v8f32)
19384 return Register();
19385 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19386 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19387 }
19388 return Register();
19389}
19390
19391Register fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19392 if (RetVT.SimpleTy != MVT::v16f32)
19393 return Register();
19394 if ((Subtarget->hasFP16())) {
19395 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
19396 }
19397 return Register();
19398}
19399
19400Register fastEmit_X86ISD_VFMULC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19401 switch (VT.SimpleTy) {
19402 case MVT::v4f32: return fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
19403 case MVT::v8f32: return fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
19404 case MVT::v16f32: return fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
19405 default: return Register();
19406 }
19407}
19408
19409// FastEmit functions for X86ISD::VFMULCSH.
19410
19411Register fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19412 if (RetVT.SimpleTy != MVT::v4f32)
19413 return Register();
19414 if ((Subtarget->hasFP16())) {
19415 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
19416 }
19417 return Register();
19418}
19419
19420Register fastEmit_X86ISD_VFMULCSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19421 switch (VT.SimpleTy) {
19422 case MVT::v4f32: return fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
19423 default: return Register();
19424 }
19425}
19426
19427// FastEmit functions for X86ISD::VFPROUND2.
19428
19429Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8f16_rr(Register Op0, Register Op1) {
19430 if ((Subtarget->hasAVX10_2())) {
19431 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19432 }
19433 return Register();
19434}
19435
19436Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8bf16_rr(Register Op0, Register Op1) {
19437 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
19438 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19439 }
19440 return Register();
19441}
19442
19443Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19444switch (RetVT.SimpleTy) {
19445 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8f16_rr(Op0, Op1);
19446 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8bf16_rr(Op0, Op1);
19447 default: return Register();
19448}
19449}
19450
19451Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16f16_rr(Register Op0, Register Op1) {
19452 if ((Subtarget->hasAVX10_2())) {
19453 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19454 }
19455 return Register();
19456}
19457
19458Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16bf16_rr(Register Op0, Register Op1) {
19459 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
19460 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19461 }
19462 return Register();
19463}
19464
19465Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19466switch (RetVT.SimpleTy) {
19467 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16f16_rr(Op0, Op1);
19468 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16bf16_rr(Op0, Op1);
19469 default: return Register();
19470}
19471}
19472
19473Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32f16_rr(Register Op0, Register Op1) {
19474 if ((Subtarget->hasAVX10_2_512())) {
19475 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZrr, RC: &X86::VR512RegClass, Op0, Op1);
19476 }
19477 return Register();
19478}
19479
19480Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32bf16_rr(Register Op0, Register Op1) {
19481 if ((Subtarget->hasBF16())) {
19482 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19483 }
19484 return Register();
19485}
19486
19487Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19488switch (RetVT.SimpleTy) {
19489 case MVT::v32f16: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32f16_rr(Op0, Op1);
19490 case MVT::v32bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32bf16_rr(Op0, Op1);
19491 default: return Register();
19492}
19493}
19494
19495Register fastEmit_X86ISD_VFPROUND2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19496 switch (VT.SimpleTy) {
19497 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_rr(RetVT, Op0, Op1);
19498 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_rr(RetVT, Op0, Op1);
19499 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_rr(RetVT, Op0, Op1);
19500 default: return Register();
19501 }
19502}
19503
19504// FastEmit functions for X86ISD::VP2INTERSECT.
19505
19506Register fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19507 if (RetVT.SimpleTy != MVT::Untyped)
19508 return Register();
19509 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19510 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ128rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
19511 }
19512 return Register();
19513}
19514
19515Register fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19516 if (RetVT.SimpleTy != MVT::Untyped)
19517 return Register();
19518 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19519 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ256rr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
19520 }
19521 return Register();
19522}
19523
19524Register fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19525 if (RetVT.SimpleTy != MVT::Untyped)
19526 return Register();
19527 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
19528 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZrr, RC: &X86::VK16PAIRRegClass, Op0, Op1);
19529 }
19530 return Register();
19531}
19532
19533Register fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19534 if (RetVT.SimpleTy != MVT::Untyped)
19535 return Register();
19536 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19537 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ128rr, RC: &X86::VK2PAIRRegClass, Op0, Op1);
19538 }
19539 return Register();
19540}
19541
19542Register fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19543 if (RetVT.SimpleTy != MVT::Untyped)
19544 return Register();
19545 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19546 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ256rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
19547 }
19548 return Register();
19549}
19550
19551Register fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19552 if (RetVT.SimpleTy != MVT::Untyped)
19553 return Register();
19554 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
19555 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZrr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
19556 }
19557 return Register();
19558}
19559
19560Register fastEmit_X86ISD_VP2INTERSECT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19561 switch (VT.SimpleTy) {
19562 case MVT::v4i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(RetVT, Op0, Op1);
19563 case MVT::v8i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(RetVT, Op0, Op1);
19564 case MVT::v16i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(RetVT, Op0, Op1);
19565 case MVT::v2i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(RetVT, Op0, Op1);
19566 case MVT::v4i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(RetVT, Op0, Op1);
19567 case MVT::v8i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(RetVT, Op0, Op1);
19568 default: return Register();
19569 }
19570}
19571
19572// FastEmit functions for X86ISD::VPERMV.
19573
19574Register fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19575 if (RetVT.SimpleTy != MVT::v16i8)
19576 return Register();
19577 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
19578 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19579 }
19580 return Register();
19581}
19582
19583Register fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19584 if (RetVT.SimpleTy != MVT::v32i8)
19585 return Register();
19586 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
19587 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19588 }
19589 return Register();
19590}
19591
19592Register fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19593 if (RetVT.SimpleTy != MVT::v64i8)
19594 return Register();
19595 if ((Subtarget->hasVBMI())) {
19596 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZrr, RC: &X86::VR512RegClass, Op0, Op1);
19597 }
19598 return Register();
19599}
19600
19601Register fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19602 if (RetVT.SimpleTy != MVT::v8i16)
19603 return Register();
19604 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19605 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19606 }
19607 return Register();
19608}
19609
19610Register fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19611 if (RetVT.SimpleTy != MVT::v16i16)
19612 return Register();
19613 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19614 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19615 }
19616 return Register();
19617}
19618
19619Register fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19620 if (RetVT.SimpleTy != MVT::v32i16)
19621 return Register();
19622 if ((Subtarget->hasBWI())) {
19623 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19624 }
19625 return Register();
19626}
19627
19628Register fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19629 if (RetVT.SimpleTy != MVT::v8i32)
19630 return Register();
19631 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19632 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19633 }
19634 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19635 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19636 }
19637 return Register();
19638}
19639
19640Register fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19641 if (RetVT.SimpleTy != MVT::v16i32)
19642 return Register();
19643 if ((Subtarget->hasAVX512())) {
19644 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19645 }
19646 return Register();
19647}
19648
19649Register fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19650 if (RetVT.SimpleTy != MVT::v4i64)
19651 return Register();
19652 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19653 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19654 }
19655 return Register();
19656}
19657
19658Register fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19659 if (RetVT.SimpleTy != MVT::v8i64)
19660 return Register();
19661 if ((Subtarget->hasAVX512())) {
19662 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19663 }
19664 return Register();
19665}
19666
19667Register fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19668 switch (VT.SimpleTy) {
19669 case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op1);
19670 case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op1);
19671 case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op1);
19672 case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op1);
19673 case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op1);
19674 case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op1);
19675 case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op1);
19676 case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op1);
19677 case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op1);
19678 case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op1);
19679 default: return Register();
19680 }
19681}
19682
19683// FastEmit functions for X86ISD::VPMADDUBSW.
19684
19685Register fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19686 if (RetVT.SimpleTy != MVT::v8i16)
19687 return Register();
19688 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19689 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19690 }
19691 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
19692 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
19693 }
19694 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19695 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
19696 }
19697 return Register();
19698}
19699
19700Register fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19701 if (RetVT.SimpleTy != MVT::v16i16)
19702 return Register();
19703 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19704 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19705 }
19706 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19707 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
19708 }
19709 return Register();
19710}
19711
19712Register fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19713 if (RetVT.SimpleTy != MVT::v32i16)
19714 return Register();
19715 if ((Subtarget->hasBWI())) {
19716 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19717 }
19718 return Register();
19719}
19720
19721Register fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19722 switch (VT.SimpleTy) {
19723 case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op1);
19724 case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op1);
19725 case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op1);
19726 default: return Register();
19727 }
19728}
19729
19730// FastEmit functions for X86ISD::VPMADDWD.
19731
19732Register fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19733 if (RetVT.SimpleTy != MVT::v4i32)
19734 return Register();
19735 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19736 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19737 }
19738 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19739 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19740 }
19741 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19742 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19743 }
19744 return Register();
19745}
19746
19747Register fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19748 if (RetVT.SimpleTy != MVT::v8i32)
19749 return Register();
19750 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19751 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19752 }
19753 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19754 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19755 }
19756 return Register();
19757}
19758
19759Register fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19760 if (RetVT.SimpleTy != MVT::v16i32)
19761 return Register();
19762 if ((Subtarget->hasBWI())) {
19763 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19764 }
19765 return Register();
19766}
19767
19768Register fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19769 switch (VT.SimpleTy) {
19770 case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op1);
19771 case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op1);
19772 case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op1);
19773 default: return Register();
19774 }
19775}
19776
19777// FastEmit functions for X86ISD::VPSHA.
19778
19779Register fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19780 if (RetVT.SimpleTy != MVT::v16i8)
19781 return Register();
19782 if ((Subtarget->hasXOP())) {
19783 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHABrr, RC: &X86::VR128RegClass, Op0, Op1);
19784 }
19785 return Register();
19786}
19787
19788Register fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19789 if (RetVT.SimpleTy != MVT::v8i16)
19790 return Register();
19791 if ((Subtarget->hasXOP())) {
19792 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAWrr, RC: &X86::VR128RegClass, Op0, Op1);
19793 }
19794 return Register();
19795}
19796
19797Register fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19798 if (RetVT.SimpleTy != MVT::v4i32)
19799 return Register();
19800 if ((Subtarget->hasXOP())) {
19801 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHADrr, RC: &X86::VR128RegClass, Op0, Op1);
19802 }
19803 return Register();
19804}
19805
19806Register fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19807 if (RetVT.SimpleTy != MVT::v2i64)
19808 return Register();
19809 if ((Subtarget->hasXOP())) {
19810 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAQrr, RC: &X86::VR128RegClass, Op0, Op1);
19811 }
19812 return Register();
19813}
19814
19815Register fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19816 switch (VT.SimpleTy) {
19817 case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op1);
19818 case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op1);
19819 case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op1);
19820 case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op1);
19821 default: return Register();
19822 }
19823}
19824
19825// FastEmit functions for X86ISD::VPSHL.
19826
19827Register fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19828 if (RetVT.SimpleTy != MVT::v16i8)
19829 return Register();
19830 if ((Subtarget->hasXOP())) {
19831 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLBrr, RC: &X86::VR128RegClass, Op0, Op1);
19832 }
19833 return Register();
19834}
19835
19836Register fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19837 if (RetVT.SimpleTy != MVT::v8i16)
19838 return Register();
19839 if ((Subtarget->hasXOP())) {
19840 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19841 }
19842 return Register();
19843}
19844
19845Register fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19846 if (RetVT.SimpleTy != MVT::v4i32)
19847 return Register();
19848 if ((Subtarget->hasXOP())) {
19849 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19850 }
19851 return Register();
19852}
19853
19854Register fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19855 if (RetVT.SimpleTy != MVT::v2i64)
19856 return Register();
19857 if ((Subtarget->hasXOP())) {
19858 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLQrr, RC: &X86::VR128RegClass, Op0, Op1);
19859 }
19860 return Register();
19861}
19862
19863Register fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19864 switch (VT.SimpleTy) {
19865 case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op1);
19866 case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
19867 case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
19868 case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
19869 default: return Register();
19870 }
19871}
19872
19873// FastEmit functions for X86ISD::VPSHUFBITQMB.
19874
19875Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19876 if (RetVT.SimpleTy != MVT::v16i1)
19877 return Register();
19878 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
19879 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ128rr, RC: &X86::VK16RegClass, Op0, Op1);
19880 }
19881 return Register();
19882}
19883
19884Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19885 if (RetVT.SimpleTy != MVT::v32i1)
19886 return Register();
19887 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
19888 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ256rr, RC: &X86::VK32RegClass, Op0, Op1);
19889 }
19890 return Register();
19891}
19892
19893Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19894 if (RetVT.SimpleTy != MVT::v64i1)
19895 return Register();
19896 if ((Subtarget->hasBITALG())) {
19897 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZrr, RC: &X86::VK64RegClass, Op0, Op1);
19898 }
19899 return Register();
19900}
19901
19902Register fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19903 switch (VT.SimpleTy) {
19904 case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op1);
19905 case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op1);
19906 case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op1);
19907 default: return Register();
19908 }
19909}
19910
19911// FastEmit functions for X86ISD::VSHL.
19912
19913Register fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19914 if (RetVT.SimpleTy != MVT::v8i16)
19915 return Register();
19916 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19917 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19918 }
19919 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19920 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19921 }
19922 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19923 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19924 }
19925 return Register();
19926}
19927
19928Register fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19929 if (RetVT.SimpleTy != MVT::v4i32)
19930 return Register();
19931 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19932 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19933 }
19934 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19935 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19936 }
19937 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19938 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19939 }
19940 return Register();
19941}
19942
19943Register fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19944 if (RetVT.SimpleTy != MVT::v2i64)
19945 return Register();
19946 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19947 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19948 }
19949 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19950 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
19951 }
19952 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19953 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
19954 }
19955 return Register();
19956}
19957
19958Register fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19959 switch (VT.SimpleTy) {
19960 case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
19961 case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
19962 case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
19963 default: return Register();
19964 }
19965}
19966
19967// FastEmit functions for X86ISD::VSHLV.
19968
19969Register fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19970 if (RetVT.SimpleTy != MVT::v8i16)
19971 return Register();
19972 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19973 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19974 }
19975 return Register();
19976}
19977
19978Register fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19979 if (RetVT.SimpleTy != MVT::v16i16)
19980 return Register();
19981 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19982 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19983 }
19984 return Register();
19985}
19986
19987Register fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19988 if (RetVT.SimpleTy != MVT::v32i16)
19989 return Register();
19990 if ((Subtarget->hasBWI())) {
19991 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19992 }
19993 return Register();
19994}
19995
19996Register fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19997 if (RetVT.SimpleTy != MVT::v4i32)
19998 return Register();
19999 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20000 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20001 }
20002 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20003 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20004 }
20005 return Register();
20006}
20007
20008Register fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20009 if (RetVT.SimpleTy != MVT::v8i32)
20010 return Register();
20011 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20012 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20013 }
20014 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20015 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20016 }
20017 return Register();
20018}
20019
20020Register fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20021 if (RetVT.SimpleTy != MVT::v16i32)
20022 return Register();
20023 if ((Subtarget->hasAVX512())) {
20024 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20025 }
20026 return Register();
20027}
20028
20029Register fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20030 if (RetVT.SimpleTy != MVT::v2i64)
20031 return Register();
20032 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20033 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20034 }
20035 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20036 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
20037 }
20038 return Register();
20039}
20040
20041Register fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20042 if (RetVT.SimpleTy != MVT::v4i64)
20043 return Register();
20044 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20045 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20046 }
20047 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20048 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
20049 }
20050 return Register();
20051}
20052
20053Register fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20054 if (RetVT.SimpleTy != MVT::v8i64)
20055 return Register();
20056 if ((Subtarget->hasAVX512())) {
20057 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20058 }
20059 return Register();
20060}
20061
20062Register fastEmit_X86ISD_VSHLV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20063 switch (VT.SimpleTy) {
20064 case MVT::v8i16: return fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(RetVT, Op0, Op1);
20065 case MVT::v16i16: return fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(RetVT, Op0, Op1);
20066 case MVT::v32i16: return fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(RetVT, Op0, Op1);
20067 case MVT::v4i32: return fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(RetVT, Op0, Op1);
20068 case MVT::v8i32: return fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(RetVT, Op0, Op1);
20069 case MVT::v16i32: return fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(RetVT, Op0, Op1);
20070 case MVT::v2i64: return fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(RetVT, Op0, Op1);
20071 case MVT::v4i64: return fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(RetVT, Op0, Op1);
20072 case MVT::v8i64: return fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(RetVT, Op0, Op1);
20073 default: return Register();
20074 }
20075}
20076
20077// FastEmit functions for X86ISD::VSRA.
20078
20079Register fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20080 if (RetVT.SimpleTy != MVT::v8i16)
20081 return Register();
20082 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20083 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20084 }
20085 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20086 return fastEmitInst_rr(MachineInstOpcode: X86::PSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
20087 }
20088 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
20089 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
20090 }
20091 return Register();
20092}
20093
20094Register fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20095 if (RetVT.SimpleTy != MVT::v4i32)
20096 return Register();
20097 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20098 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20099 }
20100 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20101 return fastEmitInst_rr(MachineInstOpcode: X86::PSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
20102 }
20103 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20104 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
20105 }
20106 return Register();
20107}
20108
20109Register fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20110 if (RetVT.SimpleTy != MVT::v2i64)
20111 return Register();
20112 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20113 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20114 }
20115 return Register();
20116}
20117
20118Register fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20119 switch (VT.SimpleTy) {
20120 case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op1);
20121 case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op1);
20122 case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op1);
20123 default: return Register();
20124 }
20125}
20126
20127// FastEmit functions for X86ISD::VSRAV.
20128
20129Register fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20130 if (RetVT.SimpleTy != MVT::v8i16)
20131 return Register();
20132 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20133 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20134 }
20135 return Register();
20136}
20137
20138Register fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20139 if (RetVT.SimpleTy != MVT::v16i16)
20140 return Register();
20141 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20142 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20143 }
20144 return Register();
20145}
20146
20147Register fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20148 if (RetVT.SimpleTy != MVT::v32i16)
20149 return Register();
20150 if ((Subtarget->hasBWI())) {
20151 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20152 }
20153 return Register();
20154}
20155
20156Register fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20157 if (RetVT.SimpleTy != MVT::v4i32)
20158 return Register();
20159 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20160 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20161 }
20162 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20163 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20164 }
20165 return Register();
20166}
20167
20168Register fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20169 if (RetVT.SimpleTy != MVT::v8i32)
20170 return Register();
20171 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20172 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20173 }
20174 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20175 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20176 }
20177 return Register();
20178}
20179
20180Register fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20181 if (RetVT.SimpleTy != MVT::v16i32)
20182 return Register();
20183 if ((Subtarget->hasAVX512())) {
20184 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20185 }
20186 return Register();
20187}
20188
20189Register fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20190 if (RetVT.SimpleTy != MVT::v2i64)
20191 return Register();
20192 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20193 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20194 }
20195 return Register();
20196}
20197
20198Register fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20199 if (RetVT.SimpleTy != MVT::v4i64)
20200 return Register();
20201 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20202 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20203 }
20204 return Register();
20205}
20206
20207Register fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20208 if (RetVT.SimpleTy != MVT::v8i64)
20209 return Register();
20210 if ((Subtarget->hasAVX512())) {
20211 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20212 }
20213 return Register();
20214}
20215
20216Register fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20217 switch (VT.SimpleTy) {
20218 case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op1);
20219 case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op1);
20220 case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op1);
20221 case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op1);
20222 case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op1);
20223 case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op1);
20224 case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op1);
20225 case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op1);
20226 case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op1);
20227 default: return Register();
20228 }
20229}
20230
20231// FastEmit functions for X86ISD::VSRL.
20232
20233Register fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20234 if (RetVT.SimpleTy != MVT::v8i16)
20235 return Register();
20236 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20237 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20238 }
20239 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20240 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20241 }
20242 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
20243 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20244 }
20245 return Register();
20246}
20247
20248Register fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20249 if (RetVT.SimpleTy != MVT::v4i32)
20250 return Register();
20251 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20252 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20253 }
20254 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20255 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20256 }
20257 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20258 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20259 }
20260 return Register();
20261}
20262
20263Register fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20264 if (RetVT.SimpleTy != MVT::v2i64)
20265 return Register();
20266 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20267 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20268 }
20269 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20270 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20271 }
20272 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20273 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20274 }
20275 return Register();
20276}
20277
20278Register fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20279 switch (VT.SimpleTy) {
20280 case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op1);
20281 case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op1);
20282 case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op1);
20283 default: return Register();
20284 }
20285}
20286
20287// FastEmit functions for X86ISD::VSRLV.
20288
20289Register fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20290 if (RetVT.SimpleTy != MVT::v8i16)
20291 return Register();
20292 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20293 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20294 }
20295 return Register();
20296}
20297
20298Register fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20299 if (RetVT.SimpleTy != MVT::v16i16)
20300 return Register();
20301 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20302 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20303 }
20304 return Register();
20305}
20306
20307Register fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20308 if (RetVT.SimpleTy != MVT::v32i16)
20309 return Register();
20310 if ((Subtarget->hasBWI())) {
20311 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20312 }
20313 return Register();
20314}
20315
20316Register fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20317 if (RetVT.SimpleTy != MVT::v4i32)
20318 return Register();
20319 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20320 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20321 }
20322 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20323 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20324 }
20325 return Register();
20326}
20327
20328Register fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20329 if (RetVT.SimpleTy != MVT::v8i32)
20330 return Register();
20331 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20332 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20333 }
20334 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20335 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20336 }
20337 return Register();
20338}
20339
20340Register fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20341 if (RetVT.SimpleTy != MVT::v16i32)
20342 return Register();
20343 if ((Subtarget->hasAVX512())) {
20344 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20345 }
20346 return Register();
20347}
20348
20349Register fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20350 if (RetVT.SimpleTy != MVT::v2i64)
20351 return Register();
20352 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20353 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20354 }
20355 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20356 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
20357 }
20358 return Register();
20359}
20360
20361Register fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20362 if (RetVT.SimpleTy != MVT::v4i64)
20363 return Register();
20364 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20365 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20366 }
20367 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20368 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
20369 }
20370 return Register();
20371}
20372
20373Register fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20374 if (RetVT.SimpleTy != MVT::v8i64)
20375 return Register();
20376 if ((Subtarget->hasAVX512())) {
20377 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20378 }
20379 return Register();
20380}
20381
20382Register fastEmit_X86ISD_VSRLV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20383 switch (VT.SimpleTy) {
20384 case MVT::v8i16: return fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(RetVT, Op0, Op1);
20385 case MVT::v16i16: return fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(RetVT, Op0, Op1);
20386 case MVT::v32i16: return fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(RetVT, Op0, Op1);
20387 case MVT::v4i32: return fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(RetVT, Op0, Op1);
20388 case MVT::v8i32: return fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(RetVT, Op0, Op1);
20389 case MVT::v16i32: return fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(RetVT, Op0, Op1);
20390 case MVT::v2i64: return fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(RetVT, Op0, Op1);
20391 case MVT::v4i64: return fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(RetVT, Op0, Op1);
20392 case MVT::v8i64: return fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(RetVT, Op0, Op1);
20393 default: return Register();
20394 }
20395}
20396
20397// Top-level FastEmit function.
20398
20399Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
20400 switch (Opcode) {
20401 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
20402 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
20403 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
20404 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
20405 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
20406 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
20407 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
20408 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
20409 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
20410 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
20411 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
20412 case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op1);
20413 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
20414 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
20415 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
20416 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
20417 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
20418 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
20419 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
20420 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
20421 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
20422 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
20423 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
20424 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
20425 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
20426 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
20427 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
20428 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
20429 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
20430 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
20431 case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op1);
20432 case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op1);
20433 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op1);
20434 case X86ISD::BT: return fastEmit_X86ISD_BT_rr(VT, RetVT, Op0, Op1);
20435 case X86ISD::BZHI: return fastEmit_X86ISD_BZHI_rr(VT, RetVT, Op0, Op1);
20436 case X86ISD::CMP: return fastEmit_X86ISD_CMP_rr(VT, RetVT, Op0, Op1);
20437 case X86ISD::COMI: return fastEmit_X86ISD_COMI_rr(VT, RetVT, Op0, Op1);
20438 case X86ISD::COMX: return fastEmit_X86ISD_COMX_rr(VT, RetVT, Op0, Op1);
20439 case X86ISD::FADDS: return fastEmit_X86ISD_FADDS_rr(VT, RetVT, Op0, Op1);
20440 case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op1);
20441 case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op1);
20442 case X86ISD::FCMP: return fastEmit_X86ISD_FCMP_rr(VT, RetVT, Op0, Op1);
20443 case X86ISD::FDIVS: return fastEmit_X86ISD_FDIVS_rr(VT, RetVT, Op0, Op1);
20444 case X86ISD::FGETEXPS: return fastEmit_X86ISD_FGETEXPS_rr(VT, RetVT, Op0, Op1);
20445 case X86ISD::FGETEXPS_SAE: return fastEmit_X86ISD_FGETEXPS_SAE_rr(VT, RetVT, Op0, Op1);
20446 case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op1);
20447 case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op1);
20448 case X86ISD::FMAX: return fastEmit_X86ISD_FMAX_rr(VT, RetVT, Op0, Op1);
20449 case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op1);
20450 case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op1);
20451 case X86ISD::FMAXS_SAE: return fastEmit_X86ISD_FMAXS_SAE_rr(VT, RetVT, Op0, Op1);
20452 case X86ISD::FMAX_SAE: return fastEmit_X86ISD_FMAX_SAE_rr(VT, RetVT, Op0, Op1);
20453 case X86ISD::FMIN: return fastEmit_X86ISD_FMIN_rr(VT, RetVT, Op0, Op1);
20454 case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op1);
20455 case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op1);
20456 case X86ISD::FMINS_SAE: return fastEmit_X86ISD_FMINS_SAE_rr(VT, RetVT, Op0, Op1);
20457 case X86ISD::FMIN_SAE: return fastEmit_X86ISD_FMIN_SAE_rr(VT, RetVT, Op0, Op1);
20458 case X86ISD::FMULS: return fastEmit_X86ISD_FMULS_rr(VT, RetVT, Op0, Op1);
20459 case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op1);
20460 case X86ISD::FP80_ADD: return fastEmit_X86ISD_FP80_ADD_rr(VT, RetVT, Op0, Op1);
20461 case X86ISD::FSQRTS: return fastEmit_X86ISD_FSQRTS_rr(VT, RetVT, Op0, Op1);
20462 case X86ISD::FSUBS: return fastEmit_X86ISD_FSUBS_rr(VT, RetVT, Op0, Op1);
20463 case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op1);
20464 case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op1);
20465 case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op1);
20466 case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op1);
20467 case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op1);
20468 case X86ISD::KORTEST: return fastEmit_X86ISD_KORTEST_rr(VT, RetVT, Op0, Op1);
20469 case X86ISD::KTEST: return fastEmit_X86ISD_KTEST_rr(VT, RetVT, Op0, Op1);
20470 case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op1);
20471 case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op1);
20472 case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op1);
20473 case X86ISD::MOVSH: return fastEmit_X86ISD_MOVSH_rr(VT, RetVT, Op0, Op1);
20474 case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op1);
20475 case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op1);
20476 case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op1);
20477 case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op1);
20478 case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op1);
20479 case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op1);
20480 case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op1);
20481 case X86ISD::PDEP: return fastEmit_X86ISD_PDEP_rr(VT, RetVT, Op0, Op1);
20482 case X86ISD::PEXT: return fastEmit_X86ISD_PEXT_rr(VT, RetVT, Op0, Op1);
20483 case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op1);
20484 case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op1);
20485 case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op1);
20486 case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op1);
20487 case X86ISD::PTEST: return fastEmit_X86ISD_PTEST_rr(VT, RetVT, Op0, Op1);
20488 case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op1);
20489 case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op1);
20490 case X86ISD::SCALEF: return fastEmit_X86ISD_SCALEF_rr(VT, RetVT, Op0, Op1);
20491 case X86ISD::SCALEFS: return fastEmit_X86ISD_SCALEFS_rr(VT, RetVT, Op0, Op1);
20492 case X86ISD::STRICT_FCMP: return fastEmit_X86ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
20493 case X86ISD::STRICT_FCMPS: return fastEmit_X86ISD_STRICT_FCMPS_rr(VT, RetVT, Op0, Op1);
20494 case X86ISD::STRICT_FMAX: return fastEmit_X86ISD_STRICT_FMAX_rr(VT, RetVT, Op0, Op1);
20495 case X86ISD::STRICT_FMIN: return fastEmit_X86ISD_STRICT_FMIN_rr(VT, RetVT, Op0, Op1);
20496 case X86ISD::STRICT_FP80_ADD: return fastEmit_X86ISD_STRICT_FP80_ADD_rr(VT, RetVT, Op0, Op1);
20497 case X86ISD::TESTP: return fastEmit_X86ISD_TESTP_rr(VT, RetVT, Op0, Op1);
20498 case X86ISD::UCOMI: return fastEmit_X86ISD_UCOMI_rr(VT, RetVT, Op0, Op1);
20499 case X86ISD::UCOMX: return fastEmit_X86ISD_UCOMX_rr(VT, RetVT, Op0, Op1);
20500 case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op1);
20501 case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op1);
20502 case X86ISD::VCVT2PH2BF8: return fastEmit_X86ISD_VCVT2PH2BF8_rr(VT, RetVT, Op0, Op1);
20503 case X86ISD::VCVT2PH2BF8S: return fastEmit_X86ISD_VCVT2PH2BF8S_rr(VT, RetVT, Op0, Op1);
20504 case X86ISD::VCVT2PH2HF8: return fastEmit_X86ISD_VCVT2PH2HF8_rr(VT, RetVT, Op0, Op1);
20505 case X86ISD::VCVT2PH2HF8S: return fastEmit_X86ISD_VCVT2PH2HF8S_rr(VT, RetVT, Op0, Op1);
20506 case X86ISD::VFCMULC: return fastEmit_X86ISD_VFCMULC_rr(VT, RetVT, Op0, Op1);
20507 case X86ISD::VFCMULCSH: return fastEmit_X86ISD_VFCMULCSH_rr(VT, RetVT, Op0, Op1);
20508 case X86ISD::VFMULC: return fastEmit_X86ISD_VFMULC_rr(VT, RetVT, Op0, Op1);
20509 case X86ISD::VFMULCSH: return fastEmit_X86ISD_VFMULCSH_rr(VT, RetVT, Op0, Op1);
20510 case X86ISD::VFPROUND2: return fastEmit_X86ISD_VFPROUND2_rr(VT, RetVT, Op0, Op1);
20511 case X86ISD::VP2INTERSECT: return fastEmit_X86ISD_VP2INTERSECT_rr(VT, RetVT, Op0, Op1);
20512 case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op1);
20513 case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op1);
20514 case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op1);
20515 case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op1);
20516 case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op1);
20517 case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op1);
20518 case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op1);
20519 case X86ISD::VSHLV: return fastEmit_X86ISD_VSHLV_rr(VT, RetVT, Op0, Op1);
20520 case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op1);
20521 case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op1);
20522 case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op1);
20523 case X86ISD::VSRLV: return fastEmit_X86ISD_VSRLV_rr(VT, RetVT, Op0, Op1);
20524 default: return Register();
20525 }
20526}
20527
20528// FastEmit functions for ISD::ADD.
20529
20530Register fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20531 if (RetVT.SimpleTy != MVT::i8)
20532 return Register();
20533 if ((Subtarget->hasNDD())) {
20534 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20535 }
20536 if ((!Subtarget->hasNDD())) {
20537 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20538 }
20539 return Register();
20540}
20541
20542Register fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20543 if (RetVT.SimpleTy != MVT::i16)
20544 return Register();
20545 if ((Subtarget->hasNDD())) {
20546 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20547 }
20548 if ((!Subtarget->hasNDD())) {
20549 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20550 }
20551 return Register();
20552}
20553
20554Register fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20555 if (RetVT.SimpleTy != MVT::i32)
20556 return Register();
20557 if ((Subtarget->hasNDD())) {
20558 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20559 }
20560 if ((!Subtarget->hasNDD())) {
20561 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20562 }
20563 return Register();
20564}
20565
20566Register fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20567 switch (VT.SimpleTy) {
20568 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, imm1);
20569 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, imm1);
20570 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, imm1);
20571 default: return Register();
20572 }
20573}
20574
20575// FastEmit functions for ISD::AND.
20576
20577Register fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20578 if (RetVT.SimpleTy != MVT::i8)
20579 return Register();
20580 if ((Subtarget->hasNDD())) {
20581 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20582 }
20583 if ((!Subtarget->hasNDD())) {
20584 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20585 }
20586 return Register();
20587}
20588
20589Register fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20590 if (RetVT.SimpleTy != MVT::i16)
20591 return Register();
20592 if ((Subtarget->hasNDD())) {
20593 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20594 }
20595 if ((!Subtarget->hasNDD())) {
20596 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20597 }
20598 return Register();
20599}
20600
20601Register fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20602 if (RetVT.SimpleTy != MVT::i32)
20603 return Register();
20604 if ((Subtarget->hasNDD())) {
20605 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20606 }
20607 if ((!Subtarget->hasNDD())) {
20608 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20609 }
20610 return Register();
20611}
20612
20613Register fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20614 switch (VT.SimpleTy) {
20615 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, imm1);
20616 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, imm1);
20617 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, imm1);
20618 default: return Register();
20619 }
20620}
20621
20622// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
20623
20624Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20625 if (RetVT.SimpleTy != MVT::i32)
20626 return Register();
20627 if ((Subtarget->hasDQI())) {
20628 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDZrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20629 }
20630 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
20631 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRDrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20632 }
20633 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
20634 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20635 }
20636 return Register();
20637}
20638
20639Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20640 if (RetVT.SimpleTy != MVT::i64)
20641 return Register();
20642 if ((Subtarget->hasDQI())) {
20643 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQZrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20644 }
20645 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
20646 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRQrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20647 }
20648 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
20649 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20650 }
20651 return Register();
20652}
20653
20654Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20655 switch (VT.SimpleTy) {
20656 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, imm1);
20657 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, imm1);
20658 default: return Register();
20659 }
20660}
20661
20662// FastEmit functions for ISD::MUL.
20663
20664Register fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20665 if (RetVT.SimpleTy != MVT::i16)
20666 return Register();
20667 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL16rri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20668}
20669
20670Register fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20671 if (RetVT.SimpleTy != MVT::i32)
20672 return Register();
20673 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL32rri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20674}
20675
20676Register fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20677 switch (VT.SimpleTy) {
20678 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, imm1);
20679 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, imm1);
20680 default: return Register();
20681 }
20682}
20683
20684// FastEmit functions for ISD::OR.
20685
20686Register fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20687 if (RetVT.SimpleTy != MVT::i8)
20688 return Register();
20689 if ((Subtarget->hasNDD())) {
20690 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20691 }
20692 if ((!Subtarget->hasNDD())) {
20693 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20694 }
20695 return Register();
20696}
20697
20698Register fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20699 if (RetVT.SimpleTy != MVT::i16)
20700 return Register();
20701 if ((Subtarget->hasNDD())) {
20702 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20703 }
20704 if ((!Subtarget->hasNDD())) {
20705 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20706 }
20707 return Register();
20708}
20709
20710Register fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20711 if (RetVT.SimpleTy != MVT::i32)
20712 return Register();
20713 if ((Subtarget->hasNDD())) {
20714 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20715 }
20716 if ((!Subtarget->hasNDD())) {
20717 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20718 }
20719 return Register();
20720}
20721
20722Register fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20723 switch (VT.SimpleTy) {
20724 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, imm1);
20725 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, imm1);
20726 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, imm1);
20727 default: return Register();
20728 }
20729}
20730
20731// FastEmit functions for ISD::ROTL.
20732
20733Register fastEmit_ISD_ROTL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20734 if (RetVT.SimpleTy != MVT::i8)
20735 return Register();
20736 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20737 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20738 }
20739 if ((!Subtarget->hasNDD())) {
20740 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20741 }
20742 return Register();
20743}
20744
20745Register fastEmit_ISD_ROTL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20746 if (RetVT.SimpleTy != MVT::i16)
20747 return Register();
20748 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20749 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20750 }
20751 if ((!Subtarget->hasNDD())) {
20752 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20753 }
20754 return Register();
20755}
20756
20757Register fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20758 if (RetVT.SimpleTy != MVT::i32)
20759 return Register();
20760 if ((Subtarget->hasFastSHLDRotate())) {
20761 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20762 }
20763 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20764 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20765 }
20766 if ((!Subtarget->hasNDD())) {
20767 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20768 }
20769 return Register();
20770}
20771
20772Register fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20773 if (RetVT.SimpleTy != MVT::i64)
20774 return Register();
20775 if ((Subtarget->hasFastSHLDRotate())) {
20776 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20777 }
20778 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20779 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20780 }
20781 if ((!Subtarget->hasNDD())) {
20782 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20783 }
20784 return Register();
20785}
20786
20787Register fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20788 switch (VT.SimpleTy) {
20789 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_ri(RetVT, Op0, imm1);
20790 case MVT::i16: return fastEmit_ISD_ROTL_MVT_i16_ri(RetVT, Op0, imm1);
20791 case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, imm1);
20792 case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, imm1);
20793 default: return Register();
20794 }
20795}
20796
20797// FastEmit functions for ISD::ROTR.
20798
20799Register fastEmit_ISD_ROTR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20800 if (RetVT.SimpleTy != MVT::i8)
20801 return Register();
20802 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20803 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20804 }
20805 if ((!Subtarget->hasNDD())) {
20806 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20807 }
20808 return Register();
20809}
20810
20811Register fastEmit_ISD_ROTR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20812 if (RetVT.SimpleTy != MVT::i16)
20813 return Register();
20814 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20815 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20816 }
20817 if ((!Subtarget->hasNDD())) {
20818 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20819 }
20820 return Register();
20821}
20822
20823Register fastEmit_ISD_ROTR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20824 if (RetVT.SimpleTy != MVT::i32)
20825 return Register();
20826 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
20827 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri_EVEX, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20828 }
20829 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
20830 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20831 }
20832 if ((Subtarget->hasFastSHLDRotate())) {
20833 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20834 }
20835 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20836 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20837 }
20838 if ((!Subtarget->hasNDD())) {
20839 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20840 }
20841 return Register();
20842}
20843
20844Register fastEmit_ISD_ROTR_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20845 if (RetVT.SimpleTy != MVT::i64)
20846 return Register();
20847 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
20848 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri_EVEX, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20849 }
20850 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
20851 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20852 }
20853 if ((Subtarget->hasFastSHLDRotate())) {
20854 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20855 }
20856 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20857 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20858 }
20859 if ((!Subtarget->hasNDD())) {
20860 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20861 }
20862 return Register();
20863}
20864
20865Register fastEmit_ISD_ROTR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20866 switch (VT.SimpleTy) {
20867 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_ri(RetVT, Op0, imm1);
20868 case MVT::i16: return fastEmit_ISD_ROTR_MVT_i16_ri(RetVT, Op0, imm1);
20869 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri(RetVT, Op0, imm1);
20870 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_ri(RetVT, Op0, imm1);
20871 default: return Register();
20872 }
20873}
20874
20875// FastEmit functions for ISD::SHL.
20876
20877Register fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20878 if (RetVT.SimpleTy != MVT::i8)
20879 return Register();
20880 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20881 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20882 }
20883 if ((!Subtarget->hasNDD())) {
20884 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20885 }
20886 return Register();
20887}
20888
20889Register fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20890 if (RetVT.SimpleTy != MVT::i16)
20891 return Register();
20892 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20893 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20894 }
20895 if ((!Subtarget->hasNDD())) {
20896 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20897 }
20898 return Register();
20899}
20900
20901Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20902 if (RetVT.SimpleTy != MVT::i32)
20903 return Register();
20904 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20905 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20906 }
20907 if ((!Subtarget->hasNDD())) {
20908 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20909 }
20910 return Register();
20911}
20912
20913Register fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20914 if (RetVT.SimpleTy != MVT::i64)
20915 return Register();
20916 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20917 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20918 }
20919 if ((!Subtarget->hasNDD())) {
20920 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20921 }
20922 return Register();
20923}
20924
20925Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20926 switch (VT.SimpleTy) {
20927 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, imm1);
20928 case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, imm1);
20929 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
20930 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, imm1);
20931 default: return Register();
20932 }
20933}
20934
20935// FastEmit functions for ISD::SRA.
20936
20937Register fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20938 if (RetVT.SimpleTy != MVT::i8)
20939 return Register();
20940 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20941 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20942 }
20943 if ((!Subtarget->hasNDD())) {
20944 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20945 }
20946 return Register();
20947}
20948
20949Register fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20950 if (RetVT.SimpleTy != MVT::i16)
20951 return Register();
20952 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20953 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20954 }
20955 if ((!Subtarget->hasNDD())) {
20956 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20957 }
20958 return Register();
20959}
20960
20961Register fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20962 if (RetVT.SimpleTy != MVT::i32)
20963 return Register();
20964 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20965 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20966 }
20967 if ((!Subtarget->hasNDD())) {
20968 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20969 }
20970 return Register();
20971}
20972
20973Register fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20974 if (RetVT.SimpleTy != MVT::i64)
20975 return Register();
20976 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20977 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20978 }
20979 if ((!Subtarget->hasNDD())) {
20980 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20981 }
20982 return Register();
20983}
20984
20985Register fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20986 switch (VT.SimpleTy) {
20987 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, imm1);
20988 case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, imm1);
20989 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, imm1);
20990 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, imm1);
20991 default: return Register();
20992 }
20993}
20994
20995// FastEmit functions for ISD::SRL.
20996
20997Register fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20998 if (RetVT.SimpleTy != MVT::i8)
20999 return Register();
21000 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21001 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21002 }
21003 if ((!Subtarget->hasNDD())) {
21004 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21005 }
21006 return Register();
21007}
21008
21009Register fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21010 if (RetVT.SimpleTy != MVT::i16)
21011 return Register();
21012 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21013 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21014 }
21015 if ((!Subtarget->hasNDD())) {
21016 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21017 }
21018 return Register();
21019}
21020
21021Register fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21022 if (RetVT.SimpleTy != MVT::i32)
21023 return Register();
21024 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21025 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21026 }
21027 if ((!Subtarget->hasNDD())) {
21028 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21029 }
21030 return Register();
21031}
21032
21033Register fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21034 if (RetVT.SimpleTy != MVT::i64)
21035 return Register();
21036 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21037 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21038 }
21039 if ((!Subtarget->hasNDD())) {
21040 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21041 }
21042 return Register();
21043}
21044
21045Register fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21046 switch (VT.SimpleTy) {
21047 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, imm1);
21048 case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, imm1);
21049 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, imm1);
21050 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, imm1);
21051 default: return Register();
21052 }
21053}
21054
21055// FastEmit functions for ISD::SUB.
21056
21057Register fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21058 if (RetVT.SimpleTy != MVT::i8)
21059 return Register();
21060 if ((Subtarget->hasNDD())) {
21061 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21062 }
21063 if ((!Subtarget->hasNDD())) {
21064 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21065 }
21066 return Register();
21067}
21068
21069Register fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21070 if (RetVT.SimpleTy != MVT::i16)
21071 return Register();
21072 if ((Subtarget->hasNDD())) {
21073 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21074 }
21075 if ((!Subtarget->hasNDD())) {
21076 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21077 }
21078 return Register();
21079}
21080
21081Register fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21082 if (RetVT.SimpleTy != MVT::i32)
21083 return Register();
21084 if ((Subtarget->hasNDD())) {
21085 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21086 }
21087 if ((!Subtarget->hasNDD())) {
21088 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21089 }
21090 return Register();
21091}
21092
21093Register fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21094 switch (VT.SimpleTy) {
21095 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, imm1);
21096 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, imm1);
21097 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, imm1);
21098 default: return Register();
21099 }
21100}
21101
21102// FastEmit functions for ISD::XOR.
21103
21104Register fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21105 if (RetVT.SimpleTy != MVT::i8)
21106 return Register();
21107 if ((Subtarget->hasNDD())) {
21108 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21109 }
21110 if ((!Subtarget->hasNDD())) {
21111 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21112 }
21113 return Register();
21114}
21115
21116Register fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21117 if (RetVT.SimpleTy != MVT::i16)
21118 return Register();
21119 if ((Subtarget->hasNDD())) {
21120 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21121 }
21122 if ((!Subtarget->hasNDD())) {
21123 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21124 }
21125 return Register();
21126}
21127
21128Register fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21129 if (RetVT.SimpleTy != MVT::i32)
21130 return Register();
21131 if ((Subtarget->hasNDD())) {
21132 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21133 }
21134 if ((!Subtarget->hasNDD())) {
21135 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21136 }
21137 return Register();
21138}
21139
21140Register fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21141 switch (VT.SimpleTy) {
21142 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, imm1);
21143 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, imm1);
21144 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, imm1);
21145 default: return Register();
21146 }
21147}
21148
21149// FastEmit functions for X86ISD::BT.
21150
21151Register fastEmit_X86ISD_BT_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21152 if (RetVT.SimpleTy != MVT::i32)
21153 return Register();
21154 return fastEmitInst_ri(MachineInstOpcode: X86::BT16ri8, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21155}
21156
21157Register fastEmit_X86ISD_BT_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21158 if (RetVT.SimpleTy != MVT::i32)
21159 return Register();
21160 return fastEmitInst_ri(MachineInstOpcode: X86::BT32ri8, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21161}
21162
21163Register fastEmit_X86ISD_BT_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21164 if (RetVT.SimpleTy != MVT::i32)
21165 return Register();
21166 return fastEmitInst_ri(MachineInstOpcode: X86::BT64ri8, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21167}
21168
21169Register fastEmit_X86ISD_BT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21170 switch (VT.SimpleTy) {
21171 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_ri(RetVT, Op0, imm1);
21172 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_ri(RetVT, Op0, imm1);
21173 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_ri(RetVT, Op0, imm1);
21174 default: return Register();
21175 }
21176}
21177
21178// Top-level FastEmit function.
21179
21180Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
21181 if (VT == MVT::i64 && Predicate_i64immSExt32(Imm: imm1))
21182 if (Register Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, imm1))
21183 return Reg;
21184
21185 switch (Opcode) {
21186 case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, imm1);
21187 case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, imm1);
21188 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
21189 case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, imm1);
21190 case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, imm1);
21191 case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, imm1);
21192 case ISD::ROTR: return fastEmit_ISD_ROTR_ri(VT, RetVT, Op0, imm1);
21193 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
21194 case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, imm1);
21195 case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, imm1);
21196 case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, imm1);
21197 case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, imm1);
21198 case X86ISD::BT: return fastEmit_X86ISD_BT_ri(VT, RetVT, Op0, imm1);
21199 default: return Register();
21200 }
21201}
21202
21203// FastEmit functions for ISD::ADD.
21204
21205Register fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21206 if (RetVT.SimpleTy != MVT::i64)
21207 return Register();
21208 if ((Subtarget->hasNDD())) {
21209 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21210 }
21211 if ((!Subtarget->hasNDD())) {
21212 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21213 }
21214 return Register();
21215}
21216
21217Register fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21218 switch (VT.SimpleTy) {
21219 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21220 default: return Register();
21221 }
21222}
21223
21224// FastEmit functions for ISD::AND.
21225
21226Register fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21227 if (RetVT.SimpleTy != MVT::i64)
21228 return Register();
21229 if ((Subtarget->hasNDD())) {
21230 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21231 }
21232 if ((!Subtarget->hasNDD())) {
21233 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21234 }
21235 return Register();
21236}
21237
21238Register fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21239 switch (VT.SimpleTy) {
21240 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21241 default: return Register();
21242 }
21243}
21244
21245// FastEmit functions for ISD::MUL.
21246
21247Register fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21248 if (RetVT.SimpleTy != MVT::i64)
21249 return Register();
21250 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL64rri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21251}
21252
21253Register fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21254 switch (VT.SimpleTy) {
21255 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21256 default: return Register();
21257 }
21258}
21259
21260// FastEmit functions for ISD::OR.
21261
21262Register fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21263 if (RetVT.SimpleTy != MVT::i64)
21264 return Register();
21265 if ((Subtarget->hasNDD())) {
21266 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21267 }
21268 if ((!Subtarget->hasNDD())) {
21269 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21270 }
21271 return Register();
21272}
21273
21274Register fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21275 switch (VT.SimpleTy) {
21276 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21277 default: return Register();
21278 }
21279}
21280
21281// FastEmit functions for ISD::SUB.
21282
21283Register fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21284 if (RetVT.SimpleTy != MVT::i64)
21285 return Register();
21286 if ((Subtarget->hasNDD())) {
21287 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21288 }
21289 if ((!Subtarget->hasNDD())) {
21290 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21291 }
21292 return Register();
21293}
21294
21295Register fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21296 switch (VT.SimpleTy) {
21297 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21298 default: return Register();
21299 }
21300}
21301
21302// FastEmit functions for ISD::XOR.
21303
21304Register fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21305 if (RetVT.SimpleTy != MVT::i64)
21306 return Register();
21307 if ((Subtarget->hasNDD())) {
21308 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21309 }
21310 if ((!Subtarget->hasNDD())) {
21311 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21312 }
21313 return Register();
21314}
21315
21316Register fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21317 switch (VT.SimpleTy) {
21318 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21319 default: return Register();
21320 }
21321}
21322
21323// Top-level FastEmit function.
21324
21325Register fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
21326 switch (Opcode) {
21327 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21328 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21329 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21330 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21331 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21332 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21333 default: return Register();
21334 }
21335}
21336
21337// FastEmit functions for ISD::Constant.
21338
21339Register fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) {
21340 if (RetVT.SimpleTy != MVT::i8)
21341 return Register();
21342 return fastEmitInst_i(MachineInstOpcode: X86::MOV8ri, RC: &X86::GR8RegClass, Imm: imm0);
21343}
21344
21345Register fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) {
21346 if (RetVT.SimpleTy != MVT::i16)
21347 return Register();
21348 return fastEmitInst_i(MachineInstOpcode: X86::MOV16ri, RC: &X86::GR16RegClass, Imm: imm0);
21349}
21350
21351Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
21352 if (RetVT.SimpleTy != MVT::i32)
21353 return Register();
21354 return fastEmitInst_i(MachineInstOpcode: X86::MOV32ri, RC: &X86::GR32RegClass, Imm: imm0);
21355}
21356
21357Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
21358 if (RetVT.SimpleTy != MVT::i64)
21359 return Register();
21360 return fastEmitInst_i(MachineInstOpcode: X86::MOV64ri, RC: &X86::GR64RegClass, Imm: imm0);
21361}
21362
21363Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
21364 switch (VT.SimpleTy) {
21365 case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0);
21366 case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0);
21367 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
21368 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
21369 default: return Register();
21370 }
21371}
21372
21373// Top-level FastEmit function.
21374
21375Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
21376 switch (Opcode) {
21377 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
21378 default: return Register();
21379 }
21380}
21381
21382