1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the X86 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_AndMask64(int64_t Imm) {
12
13 return isMask_64(Value: Imm) && !isUInt<32>(x: Imm);
14
15}
16static bool Predicate_BTRMask64(int64_t Imm) {
17
18 return !isUInt<32>(x: Imm) && !isInt<32>(x: Imm) && isPowerOf2_64(Value: ~Imm);
19
20}
21static bool Predicate_BTCBTSMask64(int64_t Imm) {
22
23 return !isInt<32>(x: Imm) && isPowerOf2_64(Value: Imm);
24
25}
26static bool Predicate_i64immSExt32(int64_t Imm) {
27 return isInt<32>(x: Imm);
28}
29
30
31// FastEmit functions for ISD::ABS.
32
33Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
34 if (RetVT.SimpleTy != MVT::v16i8)
35 return Register();
36 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
37 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ128rr, RC: &X86::VR128XRegClass, Op0);
38 }
39 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
40 return fastEmitInst_r(MachineInstOpcode: X86::PABSBrr, RC: &X86::VR128RegClass, Op0);
41 }
42 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
43 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBrr, RC: &X86::VR128RegClass, Op0);
44 }
45 return Register();
46}
47
48Register fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, Register Op0) {
49 if (RetVT.SimpleTy != MVT::v32i8)
50 return Register();
51 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
52 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ256rr, RC: &X86::VR256XRegClass, Op0);
53 }
54 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
55 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBYrr, RC: &X86::VR256RegClass, Op0);
56 }
57 return Register();
58}
59
60Register fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, Register Op0) {
61 if (RetVT.SimpleTy != MVT::v64i8)
62 return Register();
63 if ((Subtarget->hasBWI())) {
64 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZrr, RC: &X86::VR512RegClass, Op0);
65 }
66 return Register();
67}
68
69Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
70 if (RetVT.SimpleTy != MVT::v8i16)
71 return Register();
72 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
73 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ128rr, RC: &X86::VR128XRegClass, Op0);
74 }
75 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
76 return fastEmitInst_r(MachineInstOpcode: X86::PABSWrr, RC: &X86::VR128RegClass, Op0);
77 }
78 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
79 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWrr, RC: &X86::VR128RegClass, Op0);
80 }
81 return Register();
82}
83
84Register fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, Register Op0) {
85 if (RetVT.SimpleTy != MVT::v16i16)
86 return Register();
87 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
88 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ256rr, RC: &X86::VR256XRegClass, Op0);
89 }
90 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
91 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWYrr, RC: &X86::VR256RegClass, Op0);
92 }
93 return Register();
94}
95
96Register fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, Register Op0) {
97 if (RetVT.SimpleTy != MVT::v32i16)
98 return Register();
99 if ((Subtarget->hasBWI())) {
100 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZrr, RC: &X86::VR512RegClass, Op0);
101 }
102 return Register();
103}
104
105Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
106 if (RetVT.SimpleTy != MVT::v4i32)
107 return Register();
108 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
109 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ128rr, RC: &X86::VR128XRegClass, Op0);
110 }
111 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
112 return fastEmitInst_r(MachineInstOpcode: X86::PABSDrr, RC: &X86::VR128RegClass, Op0);
113 }
114 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
115 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDrr, RC: &X86::VR128RegClass, Op0);
116 }
117 return Register();
118}
119
120Register fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, Register Op0) {
121 if (RetVT.SimpleTy != MVT::v8i32)
122 return Register();
123 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
124 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ256rr, RC: &X86::VR256XRegClass, Op0);
125 }
126 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
127 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDYrr, RC: &X86::VR256RegClass, Op0);
128 }
129 return Register();
130}
131
132Register fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, Register Op0) {
133 if (RetVT.SimpleTy != MVT::v16i32)
134 return Register();
135 if ((Subtarget->hasAVX512())) {
136 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZrr, RC: &X86::VR512RegClass, Op0);
137 }
138 return Register();
139}
140
141Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
142 if (RetVT.SimpleTy != MVT::v2i64)
143 return Register();
144 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
145 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ128rr, RC: &X86::VR128XRegClass, Op0);
146 }
147 return Register();
148}
149
150Register fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, Register Op0) {
151 if (RetVT.SimpleTy != MVT::v4i64)
152 return Register();
153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
154 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ256rr, RC: &X86::VR256XRegClass, Op0);
155 }
156 return Register();
157}
158
159Register fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, Register Op0) {
160 if (RetVT.SimpleTy != MVT::v8i64)
161 return Register();
162 if ((Subtarget->hasAVX512())) {
163 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZrr, RC: &X86::VR512RegClass, Op0);
164 }
165 return Register();
166}
167
168Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
169 switch (VT.SimpleTy) {
170 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
171 case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0);
172 case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0);
173 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
174 case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0);
175 case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0);
176 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
177 case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0);
178 case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0);
179 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
180 case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0);
181 case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0);
182 default: return Register();
183 }
184}
185
186// FastEmit functions for ISD::ANY_EXTEND.
187
188Register fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
189 if (RetVT.SimpleTy != MVT::i32)
190 return Register();
191 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
192}
193
194Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
195 switch (VT.SimpleTy) {
196 case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0);
197 default: return Register();
198 }
199}
200
201// FastEmit functions for ISD::BITCAST.
202
203Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
204 if (RetVT.SimpleTy != MVT::f32)
205 return Register();
206 if ((Subtarget->hasAVX512())) {
207 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSZrr, RC: &X86::FR32XRegClass, Op0);
208 }
209 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
210 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
211 }
212 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
213 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
214 }
215 return Register();
216}
217
218Register fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Register Op0) {
219 if ((Subtarget->hasAVX512())) {
220 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDZrr, RC: &X86::FR64XRegClass, Op0);
221 }
222 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
223 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toSDrr, RC: &X86::FR64RegClass, Op0);
224 }
225 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
226 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDrr, RC: &X86::FR64RegClass, Op0);
227 }
228 return Register();
229}
230
231Register fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Register Op0) {
232 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
233 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64to64rr, RC: &X86::VR64RegClass, Op0);
234 }
235 return Register();
236}
237
238Register fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, Register Op0) {
239switch (RetVT.SimpleTy) {
240 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0);
241 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0);
242 default: return Register();
243}
244}
245
246Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
247 if (RetVT.SimpleTy != MVT::i32)
248 return Register();
249 if ((Subtarget->hasAVX512())) {
250 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIZrr, RC: &X86::GR32RegClass, Op0);
251 }
252 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
253 return fastEmitInst_r(MachineInstOpcode: X86::MOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
254 }
255 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
256 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
257 }
258 return Register();
259}
260
261Register fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Register Op0) {
262 if ((Subtarget->hasAVX512())) {
263 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64Zrr, RC: &X86::GR64RegClass, Op0);
264 }
265 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
266 return fastEmitInst_r(MachineInstOpcode: X86::MOVSDto64rr, RC: &X86::GR64RegClass, Op0);
267 }
268 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
269 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64rr, RC: &X86::GR64RegClass, Op0);
270 }
271 return Register();
272}
273
274Register fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Register Op0) {
275 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
276 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVFR642Qrr, RC: &X86::VR64RegClass, Op0);
277 }
278 return Register();
279}
280
281Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
282switch (RetVT.SimpleTy) {
283 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0);
284 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0);
285 default: return Register();
286}
287}
288
289Register fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Register Op0) {
290 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
291 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64from64rr, RC: &X86::GR64RegClass, Op0);
292 }
293 return Register();
294}
295
296Register fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Register Op0) {
297 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
298 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2FR64rr, RC: &X86::FR64RegClass, Op0);
299 }
300 return Register();
301}
302
303Register fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, Register Op0) {
304switch (RetVT.SimpleTy) {
305 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0);
306 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0);
307 default: return Register();
308}
309}
310
311Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
312 switch (VT.SimpleTy) {
313 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
314 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0);
315 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
316 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
317 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0);
318 default: return Register();
319 }
320}
321
322// FastEmit functions for ISD::BITREVERSE.
323
324Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
325 if (RetVT.SimpleTy != MVT::v16i8)
326 return Register();
327 if ((Subtarget->hasBMM()) && (Subtarget->hasVLX())) {
328 return fastEmitInst_r(MachineInstOpcode: X86::VPBITREVBZ128rr, RC: &X86::VR128XRegClass, Op0);
329 }
330 return Register();
331}
332
333Register fastEmit_ISD_BITREVERSE_MVT_v32i8_r(MVT RetVT, Register Op0) {
334 if (RetVT.SimpleTy != MVT::v32i8)
335 return Register();
336 if ((Subtarget->hasBMM()) && (Subtarget->hasVLX())) {
337 return fastEmitInst_r(MachineInstOpcode: X86::VPBITREVBZ256rr, RC: &X86::VR256XRegClass, Op0);
338 }
339 return Register();
340}
341
342Register fastEmit_ISD_BITREVERSE_MVT_v64i8_r(MVT RetVT, Register Op0) {
343 if (RetVT.SimpleTy != MVT::v64i8)
344 return Register();
345 if ((Subtarget->hasBMM())) {
346 return fastEmitInst_r(MachineInstOpcode: X86::VPBITREVBZrr, RC: &X86::VR512RegClass, Op0);
347 }
348 return Register();
349}
350
351Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
352 switch (VT.SimpleTy) {
353 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
354 case MVT::v32i8: return fastEmit_ISD_BITREVERSE_MVT_v32i8_r(RetVT, Op0);
355 case MVT::v64i8: return fastEmit_ISD_BITREVERSE_MVT_v64i8_r(RetVT, Op0);
356 default: return Register();
357 }
358}
359
360// FastEmit functions for ISD::BSWAP.
361
362Register fastEmit_ISD_BSWAP_MVT_i16_r(MVT RetVT, Register Op0) {
363 if (RetVT.SimpleTy != MVT::i16)
364 return Register();
365 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
366 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE16rr, RC: &X86::GR16RegClass, Op0);
367 }
368 return Register();
369}
370
371Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
372 if (RetVT.SimpleTy != MVT::i32)
373 return Register();
374 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
375 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE32rr, RC: &X86::GR32RegClass, Op0);
376 }
377 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
378 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP32r, RC: &X86::GR32RegClass, Op0);
379 }
380 return Register();
381}
382
383Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
384 if (RetVT.SimpleTy != MVT::i64)
385 return Register();
386 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
387 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE64rr, RC: &X86::GR64RegClass, Op0);
388 }
389 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
390 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP64r, RC: &X86::GR64RegClass, Op0);
391 }
392 return Register();
393}
394
395Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
396 switch (VT.SimpleTy) {
397 case MVT::i16: return fastEmit_ISD_BSWAP_MVT_i16_r(RetVT, Op0);
398 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
399 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
400 default: return Register();
401 }
402}
403
404// FastEmit functions for ISD::CTLZ.
405
406Register fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, Register Op0) {
407 if (RetVT.SimpleTy != MVT::i16)
408 return Register();
409 if ((Subtarget->hasLZCNT())) {
410 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT16rr, RC: &X86::GR16RegClass, Op0);
411 }
412 return Register();
413}
414
415Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
416 if (RetVT.SimpleTy != MVT::i32)
417 return Register();
418 if ((Subtarget->hasLZCNT())) {
419 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT32rr, RC: &X86::GR32RegClass, Op0);
420 }
421 return Register();
422}
423
424Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
425 if (RetVT.SimpleTy != MVT::i64)
426 return Register();
427 if ((Subtarget->hasLZCNT())) {
428 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT64rr, RC: &X86::GR64RegClass, Op0);
429 }
430 return Register();
431}
432
433Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
434 if (RetVT.SimpleTy != MVT::v4i32)
435 return Register();
436 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
437 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
438 }
439 return Register();
440}
441
442Register fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, Register Op0) {
443 if (RetVT.SimpleTy != MVT::v8i32)
444 return Register();
445 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
446 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
447 }
448 return Register();
449}
450
451Register fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, Register Op0) {
452 if (RetVT.SimpleTy != MVT::v16i32)
453 return Register();
454 if ((Subtarget->hasCDI())) {
455 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZrr, RC: &X86::VR512RegClass, Op0);
456 }
457 return Register();
458}
459
460Register fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, Register Op0) {
461 if (RetVT.SimpleTy != MVT::v2i64)
462 return Register();
463 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
464 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
465 }
466 return Register();
467}
468
469Register fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, Register Op0) {
470 if (RetVT.SimpleTy != MVT::v4i64)
471 return Register();
472 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
473 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
474 }
475 return Register();
476}
477
478Register fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, Register Op0) {
479 if (RetVT.SimpleTy != MVT::v8i64)
480 return Register();
481 if ((Subtarget->hasCDI())) {
482 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZrr, RC: &X86::VR512RegClass, Op0);
483 }
484 return Register();
485}
486
487Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
488 switch (VT.SimpleTy) {
489 case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0);
490 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
491 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
492 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
493 case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0);
494 case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0);
495 case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0);
496 case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0);
497 case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0);
498 default: return Register();
499 }
500}
501
502// FastEmit functions for ISD::CTPOP.
503
504Register fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, Register Op0) {
505 if (RetVT.SimpleTy != MVT::i16)
506 return Register();
507 if ((Subtarget->hasPOPCNT())) {
508 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT16rr, RC: &X86::GR16RegClass, Op0);
509 }
510 return Register();
511}
512
513Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
514 if (RetVT.SimpleTy != MVT::i32)
515 return Register();
516 if ((Subtarget->hasPOPCNT())) {
517 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT32rr, RC: &X86::GR32RegClass, Op0);
518 }
519 return Register();
520}
521
522Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
523 if (RetVT.SimpleTy != MVT::i64)
524 return Register();
525 if ((Subtarget->hasPOPCNT())) {
526 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT64rr, RC: &X86::GR64RegClass, Op0);
527 }
528 return Register();
529}
530
531Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
532 if (RetVT.SimpleTy != MVT::v16i8)
533 return Register();
534 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
535 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ128rr, RC: &X86::VR128XRegClass, Op0);
536 }
537 return Register();
538}
539
540Register fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, Register Op0) {
541 if (RetVT.SimpleTy != MVT::v32i8)
542 return Register();
543 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
544 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ256rr, RC: &X86::VR256XRegClass, Op0);
545 }
546 return Register();
547}
548
549Register fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, Register Op0) {
550 if (RetVT.SimpleTy != MVT::v64i8)
551 return Register();
552 if ((Subtarget->hasBITALG())) {
553 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZrr, RC: &X86::VR512RegClass, Op0);
554 }
555 return Register();
556}
557
558Register fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, Register Op0) {
559 if (RetVT.SimpleTy != MVT::v8i16)
560 return Register();
561 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
562 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ128rr, RC: &X86::VR128XRegClass, Op0);
563 }
564 return Register();
565}
566
567Register fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, Register Op0) {
568 if (RetVT.SimpleTy != MVT::v16i16)
569 return Register();
570 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
571 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ256rr, RC: &X86::VR256XRegClass, Op0);
572 }
573 return Register();
574}
575
576Register fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, Register Op0) {
577 if (RetVT.SimpleTy != MVT::v32i16)
578 return Register();
579 if ((Subtarget->hasBITALG())) {
580 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZrr, RC: &X86::VR512RegClass, Op0);
581 }
582 return Register();
583}
584
585Register fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, Register Op0) {
586 if (RetVT.SimpleTy != MVT::v4i32)
587 return Register();
588 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
589 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
590 }
591 return Register();
592}
593
594Register fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, Register Op0) {
595 if (RetVT.SimpleTy != MVT::v8i32)
596 return Register();
597 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
598 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
599 }
600 return Register();
601}
602
603Register fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, Register Op0) {
604 if (RetVT.SimpleTy != MVT::v16i32)
605 return Register();
606 if ((Subtarget->hasVPOPCNTDQ())) {
607 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZrr, RC: &X86::VR512RegClass, Op0);
608 }
609 return Register();
610}
611
612Register fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, Register Op0) {
613 if (RetVT.SimpleTy != MVT::v2i64)
614 return Register();
615 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
616 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
617 }
618 return Register();
619}
620
621Register fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, Register Op0) {
622 if (RetVT.SimpleTy != MVT::v4i64)
623 return Register();
624 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
625 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
626 }
627 return Register();
628}
629
630Register fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, Register Op0) {
631 if (RetVT.SimpleTy != MVT::v8i64)
632 return Register();
633 if ((Subtarget->hasVPOPCNTDQ())) {
634 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZrr, RC: &X86::VR512RegClass, Op0);
635 }
636 return Register();
637}
638
639Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
640 switch (VT.SimpleTy) {
641 case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0);
642 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
643 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
644 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
645 case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0);
646 case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0);
647 case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0);
648 case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0);
649 case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0);
650 case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0);
651 case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0);
652 case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0);
653 case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0);
654 case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0);
655 case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0);
656 default: return Register();
657 }
658}
659
660// FastEmit functions for ISD::CTTZ.
661
662Register fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, Register Op0) {
663 if (RetVT.SimpleTy != MVT::i16)
664 return Register();
665 if ((Subtarget->hasBMI())) {
666 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT16rr, RC: &X86::GR16RegClass, Op0);
667 }
668 return Register();
669}
670
671Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
672 if (RetVT.SimpleTy != MVT::i32)
673 return Register();
674 if ((Subtarget->hasBMI())) {
675 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT32rr, RC: &X86::GR32RegClass, Op0);
676 }
677 return Register();
678}
679
680Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
681 if (RetVT.SimpleTy != MVT::i64)
682 return Register();
683 if ((Subtarget->hasBMI())) {
684 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT64rr, RC: &X86::GR64RegClass, Op0);
685 }
686 return Register();
687}
688
689Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
690 switch (VT.SimpleTy) {
691 case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0);
692 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
693 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
694 default: return Register();
695 }
696}
697
698// FastEmit functions for ISD::FABS.
699
700Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
701 if (RetVT.SimpleTy != MVT::f32)
702 return Register();
703 if ((!Subtarget->hasSSE1())) {
704 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp32, RC: &X86::RFP32RegClass, Op0);
705 }
706 return Register();
707}
708
709Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
710 if (RetVT.SimpleTy != MVT::f64)
711 return Register();
712 if ((!Subtarget->hasSSE2())) {
713 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp64, RC: &X86::RFP64RegClass, Op0);
714 }
715 return Register();
716}
717
718Register fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, Register Op0) {
719 if (RetVT.SimpleTy != MVT::f80)
720 return Register();
721 if ((Subtarget->hasX87())) {
722 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp80, RC: &X86::RFP80RegClass, Op0);
723 }
724 return Register();
725}
726
727Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
728 switch (VT.SimpleTy) {
729 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
730 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
731 case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0);
732 default: return Register();
733 }
734}
735
736// FastEmit functions for ISD::FNEG.
737
738Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
739 if (RetVT.SimpleTy != MVT::f32)
740 return Register();
741 if ((!Subtarget->hasSSE1())) {
742 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp32, RC: &X86::RFP32RegClass, Op0);
743 }
744 return Register();
745}
746
747Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
748 if (RetVT.SimpleTy != MVT::f64)
749 return Register();
750 if ((!Subtarget->hasSSE2())) {
751 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp64, RC: &X86::RFP64RegClass, Op0);
752 }
753 return Register();
754}
755
756Register fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, Register Op0) {
757 if (RetVT.SimpleTy != MVT::f80)
758 return Register();
759 if ((Subtarget->hasX87())) {
760 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp80, RC: &X86::RFP80RegClass, Op0);
761 }
762 return Register();
763}
764
765Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
766 switch (VT.SimpleTy) {
767 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
768 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
769 case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0);
770 default: return Register();
771 }
772}
773
774// FastEmit functions for ISD::FP_EXTEND.
775
776Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
777 if (RetVT.SimpleTy != MVT::f64)
778 return Register();
779 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
780 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
781 }
782 return Register();
783}
784
785Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Register Op0) {
786 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
787 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
788 }
789 return Register();
790}
791
792Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Register Op0) {
793 if ((Subtarget->hasFP16())) {
794 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
795 }
796 return Register();
797}
798
799Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(MVT RetVT, Register Op0) {
800switch (RetVT.SimpleTy) {
801 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
802 case MVT::v8f64: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
803 default: return Register();
804}
805}
806
807Register fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(MVT RetVT, Register Op0) {
808 if (RetVT.SimpleTy != MVT::v16f32)
809 return Register();
810 if ((Subtarget->hasFP16())) {
811 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
812 }
813 return Register();
814}
815
816Register fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, Register Op0) {
817 if (RetVT.SimpleTy != MVT::v4f64)
818 return Register();
819 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
820 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
821 }
822 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
823 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
824 }
825 return Register();
826}
827
828Register fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, Register Op0) {
829 if (RetVT.SimpleTy != MVT::v8f64)
830 return Register();
831 if ((Subtarget->hasAVX512())) {
832 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
833 }
834 return Register();
835}
836
837Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
838 switch (VT.SimpleTy) {
839 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
840 case MVT::v8f16: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
841 case MVT::v16f16: return fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
842 case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
843 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
844 default: return Register();
845 }
846}
847
848// FastEmit functions for ISD::FP_ROUND.
849
850Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
851 if (RetVT.SimpleTy != MVT::f32)
852 return Register();
853 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
854 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
855 }
856 return Register();
857}
858
859Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
860 switch (VT.SimpleTy) {
861 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
862 default: return Register();
863 }
864}
865
866// FastEmit functions for ISD::FP_TO_SINT.
867
868Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
869 if ((Subtarget->hasFP16())) {
870 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
871 }
872 return Register();
873}
874
875Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
876 if ((Subtarget->hasFP16())) {
877 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
878 }
879 return Register();
880}
881
882Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
883switch (RetVT.SimpleTy) {
884 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
885 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
886 default: return Register();
887}
888}
889
890Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
891 if ((Subtarget->hasAVX512())) {
892 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
893 }
894 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
895 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
896 }
897 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
898 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
899 }
900 return Register();
901}
902
903Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
904 if ((Subtarget->hasAVX512())) {
905 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
906 }
907 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
908 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
909 }
910 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
911 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
912 }
913 return Register();
914}
915
916Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
917switch (RetVT.SimpleTy) {
918 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
919 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
920 default: return Register();
921}
922}
923
924Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
925 if ((Subtarget->hasAVX512())) {
926 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
927 }
928 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
929 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
930 }
931 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
932 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
933 }
934 return Register();
935}
936
937Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
938 if ((Subtarget->hasAVX512())) {
939 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
940 }
941 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
942 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
943 }
944 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
945 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
946 }
947 return Register();
948}
949
950Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
951switch (RetVT.SimpleTy) {
952 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
953 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
954 default: return Register();
955}
956}
957
958Register fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
959 if (RetVT.SimpleTy != MVT::v4i32)
960 return Register();
961 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
963 }
964 return Register();
965}
966
967Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
968 switch (VT.SimpleTy) {
969 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
970 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
971 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
972 case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
973 default: return Register();
974 }
975}
976
977// FastEmit functions for ISD::FP_TO_UINT.
978
979Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
980 if ((Subtarget->hasFP16())) {
981 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
982 }
983 return Register();
984}
985
986Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
987 if ((Subtarget->hasFP16())) {
988 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
989 }
990 return Register();
991}
992
993Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
994switch (RetVT.SimpleTy) {
995 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
996 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
997 default: return Register();
998}
999}
1000
1001Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
1002 if ((Subtarget->hasAVX512())) {
1003 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
1004 }
1005 return Register();
1006}
1007
1008Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
1009 if ((Subtarget->hasAVX512())) {
1010 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1011 }
1012 return Register();
1013}
1014
1015Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
1016switch (RetVT.SimpleTy) {
1017 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
1018 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
1019 default: return Register();
1020}
1021}
1022
1023Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
1024 if ((Subtarget->hasAVX512())) {
1025 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
1026 }
1027 return Register();
1028}
1029
1030Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
1031 if ((Subtarget->hasAVX512())) {
1032 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1033 }
1034 return Register();
1035}
1036
1037Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
1038switch (RetVT.SimpleTy) {
1039 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
1040 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
1041 default: return Register();
1042}
1043}
1044
1045Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
1046 switch (VT.SimpleTy) {
1047 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
1048 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
1049 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
1050 default: return Register();
1051 }
1052}
1053
1054// FastEmit functions for ISD::FSQRT.
1055
1056Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
1057 if (RetVT.SimpleTy != MVT::f32)
1058 return Register();
1059 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1060 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
1061 }
1062 if ((!Subtarget->hasSSE1())) {
1063 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
1064 }
1065 return Register();
1066}
1067
1068Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
1069 if (RetVT.SimpleTy != MVT::f64)
1070 return Register();
1071 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1072 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
1073 }
1074 if ((!Subtarget->hasSSE2())) {
1075 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
1076 }
1077 return Register();
1078}
1079
1080Register fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, Register Op0) {
1081 if (RetVT.SimpleTy != MVT::f80)
1082 return Register();
1083 if ((Subtarget->hasX87())) {
1084 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
1085 }
1086 return Register();
1087}
1088
1089Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1090 if (RetVT.SimpleTy != MVT::v8f16)
1091 return Register();
1092 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1093 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
1094 }
1095 return Register();
1096}
1097
1098Register fastEmit_ISD_FSQRT_MVT_v16f16_r(MVT RetVT, Register Op0) {
1099 if (RetVT.SimpleTy != MVT::v16f16)
1100 return Register();
1101 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1102 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
1103 }
1104 return Register();
1105}
1106
1107Register fastEmit_ISD_FSQRT_MVT_v32f16_r(MVT RetVT, Register Op0) {
1108 if (RetVT.SimpleTy != MVT::v32f16)
1109 return Register();
1110 if ((Subtarget->hasFP16())) {
1111 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
1112 }
1113 return Register();
1114}
1115
1116Register fastEmit_ISD_FSQRT_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1117 if (RetVT.SimpleTy != MVT::v8bf16)
1118 return Register();
1119 if ((Subtarget->hasAVX10_2())) {
1120 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
1121 }
1122 return Register();
1123}
1124
1125Register fastEmit_ISD_FSQRT_MVT_v16bf16_r(MVT RetVT, Register Op0) {
1126 if (RetVT.SimpleTy != MVT::v16bf16)
1127 return Register();
1128 if ((Subtarget->hasAVX10_2())) {
1129 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
1130 }
1131 return Register();
1132}
1133
1134Register fastEmit_ISD_FSQRT_MVT_v32bf16_r(MVT RetVT, Register Op0) {
1135 if (RetVT.SimpleTy != MVT::v32bf16)
1136 return Register();
1137 if ((Subtarget->hasAVX10_2())) {
1138 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
1139 }
1140 return Register();
1141}
1142
1143Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1144 if (RetVT.SimpleTy != MVT::v4f32)
1145 return Register();
1146 if ((Subtarget->hasVLX())) {
1147 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
1148 }
1149 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1150 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
1151 }
1152 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1153 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
1154 }
1155 return Register();
1156}
1157
1158Register fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1159 if (RetVT.SimpleTy != MVT::v8f32)
1160 return Register();
1161 if ((Subtarget->hasVLX())) {
1162 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
1163 }
1164 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1165 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
1166 }
1167 return Register();
1168}
1169
1170Register fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, Register Op0) {
1171 if (RetVT.SimpleTy != MVT::v16f32)
1172 return Register();
1173 if ((Subtarget->hasAVX512())) {
1174 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
1175 }
1176 return Register();
1177}
1178
1179Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1180 if (RetVT.SimpleTy != MVT::v2f64)
1181 return Register();
1182 if ((Subtarget->hasVLX())) {
1183 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
1184 }
1185 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1186 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
1187 }
1188 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1189 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
1190 }
1191 return Register();
1192}
1193
1194Register fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1195 if (RetVT.SimpleTy != MVT::v4f64)
1196 return Register();
1197 if ((Subtarget->hasVLX())) {
1198 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
1199 }
1200 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1201 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
1202 }
1203 return Register();
1204}
1205
1206Register fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1207 if (RetVT.SimpleTy != MVT::v8f64)
1208 return Register();
1209 if ((Subtarget->hasAVX512())) {
1210 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
1211 }
1212 return Register();
1213}
1214
1215Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
1216 switch (VT.SimpleTy) {
1217 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
1218 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
1219 case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0);
1220 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
1221 case MVT::v16f16: return fastEmit_ISD_FSQRT_MVT_v16f16_r(RetVT, Op0);
1222 case MVT::v32f16: return fastEmit_ISD_FSQRT_MVT_v32f16_r(RetVT, Op0);
1223 case MVT::v8bf16: return fastEmit_ISD_FSQRT_MVT_v8bf16_r(RetVT, Op0);
1224 case MVT::v16bf16: return fastEmit_ISD_FSQRT_MVT_v16bf16_r(RetVT, Op0);
1225 case MVT::v32bf16: return fastEmit_ISD_FSQRT_MVT_v32bf16_r(RetVT, Op0);
1226 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
1227 case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0);
1228 case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0);
1229 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
1230 case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0);
1231 case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0);
1232 default: return Register();
1233 }
1234}
1235
1236// FastEmit functions for ISD::LLRINT.
1237
1238Register fastEmit_ISD_LLRINT_MVT_f16_r(MVT RetVT, Register Op0) {
1239 if (RetVT.SimpleTy != MVT::i64)
1240 return Register();
1241 if ((Subtarget->hasFP16()) && (Subtarget->is64Bit())) {
1242 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1243 }
1244 return Register();
1245}
1246
1247Register fastEmit_ISD_LLRINT_MVT_f32_r(MVT RetVT, Register Op0) {
1248 if (RetVT.SimpleTy != MVT::i64)
1249 return Register();
1250 if ((Subtarget->hasAVX512())) {
1251 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1252 }
1253 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1254 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1255 }
1256 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1257 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1258 }
1259 return Register();
1260}
1261
1262Register fastEmit_ISD_LLRINT_MVT_f64_r(MVT RetVT, Register Op0) {
1263 if (RetVT.SimpleTy != MVT::i64)
1264 return Register();
1265 if ((Subtarget->hasAVX512())) {
1266 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1267 }
1268 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1269 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1270 }
1271 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1272 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1273 }
1274 return Register();
1275}
1276
1277Register fastEmit_ISD_LLRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1278 if (RetVT.SimpleTy != MVT::v8i64)
1279 return Register();
1280 if ((Subtarget->hasFP16())) {
1281 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
1282 }
1283 return Register();
1284}
1285
1286Register fastEmit_ISD_LLRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1287 if (RetVT.SimpleTy != MVT::v4i64)
1288 return Register();
1289 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1290 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1291 }
1292 return Register();
1293}
1294
1295Register fastEmit_ISD_LLRINT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1296 if (RetVT.SimpleTy != MVT::v8i64)
1297 return Register();
1298 if ((Subtarget->hasDQI())) {
1299 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1300 }
1301 return Register();
1302}
1303
1304Register fastEmit_ISD_LLRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1305 if (RetVT.SimpleTy != MVT::v2i64)
1306 return Register();
1307 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1308 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1309 }
1310 return Register();
1311}
1312
1313Register fastEmit_ISD_LLRINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1314 if (RetVT.SimpleTy != MVT::v4i64)
1315 return Register();
1316 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1317 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1318 }
1319 return Register();
1320}
1321
1322Register fastEmit_ISD_LLRINT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1323 if (RetVT.SimpleTy != MVT::v8i64)
1324 return Register();
1325 if ((Subtarget->hasDQI())) {
1326 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1327 }
1328 return Register();
1329}
1330
1331Register fastEmit_ISD_LLRINT_r(MVT VT, MVT RetVT, Register Op0) {
1332 switch (VT.SimpleTy) {
1333 case MVT::f16: return fastEmit_ISD_LLRINT_MVT_f16_r(RetVT, Op0);
1334 case MVT::f32: return fastEmit_ISD_LLRINT_MVT_f32_r(RetVT, Op0);
1335 case MVT::f64: return fastEmit_ISD_LLRINT_MVT_f64_r(RetVT, Op0);
1336 case MVT::v8f16: return fastEmit_ISD_LLRINT_MVT_v8f16_r(RetVT, Op0);
1337 case MVT::v4f32: return fastEmit_ISD_LLRINT_MVT_v4f32_r(RetVT, Op0);
1338 case MVT::v8f32: return fastEmit_ISD_LLRINT_MVT_v8f32_r(RetVT, Op0);
1339 case MVT::v2f64: return fastEmit_ISD_LLRINT_MVT_v2f64_r(RetVT, Op0);
1340 case MVT::v4f64: return fastEmit_ISD_LLRINT_MVT_v4f64_r(RetVT, Op0);
1341 case MVT::v8f64: return fastEmit_ISD_LLRINT_MVT_v8f64_r(RetVT, Op0);
1342 default: return Register();
1343 }
1344}
1345
1346// FastEmit functions for ISD::LRINT.
1347
1348Register fastEmit_ISD_LRINT_MVT_f16_MVT_i32_r(Register Op0) {
1349 if ((Subtarget->hasFP16())) {
1350 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
1351 }
1352 return Register();
1353}
1354
1355Register fastEmit_ISD_LRINT_MVT_f16_MVT_i64_r(Register Op0) {
1356 if ((Subtarget->hasFP16()) && (Subtarget->is64Bit())) {
1357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1358 }
1359 return Register();
1360}
1361
1362Register fastEmit_ISD_LRINT_MVT_f16_r(MVT RetVT, Register Op0) {
1363switch (RetVT.SimpleTy) {
1364 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f16_MVT_i32_r(Op0);
1365 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f16_MVT_i64_r(Op0);
1366 default: return Register();
1367}
1368}
1369
1370Register fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Register Op0) {
1371 if ((Subtarget->hasAVX512())) {
1372 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
1373 }
1374 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1375 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1376 }
1377 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1378 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1379 }
1380 return Register();
1381}
1382
1383Register fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Register Op0) {
1384 if ((Subtarget->hasAVX512())) {
1385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1386 }
1387 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1388 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1389 }
1390 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1391 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1392 }
1393 return Register();
1394}
1395
1396Register fastEmit_ISD_LRINT_MVT_f32_r(MVT RetVT, Register Op0) {
1397switch (RetVT.SimpleTy) {
1398 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Op0);
1399 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Op0);
1400 default: return Register();
1401}
1402}
1403
1404Register fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Register Op0) {
1405 if ((Subtarget->hasAVX512())) {
1406 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
1407 }
1408 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1409 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1410 }
1411 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1412 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1413 }
1414 return Register();
1415}
1416
1417Register fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Register Op0) {
1418 if ((Subtarget->hasAVX512())) {
1419 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1420 }
1421 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1422 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1423 }
1424 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1425 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1426 }
1427 return Register();
1428}
1429
1430Register fastEmit_ISD_LRINT_MVT_f64_r(MVT RetVT, Register Op0) {
1431switch (RetVT.SimpleTy) {
1432 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Op0);
1433 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Op0);
1434 default: return Register();
1435}
1436}
1437
1438Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i16_r(Register Op0) {
1439 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1440 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
1441 }
1442 return Register();
1443}
1444
1445Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i32_r(Register Op0) {
1446 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1447 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1448 }
1449 return Register();
1450}
1451
1452Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i64_r(Register Op0) {
1453 if ((Subtarget->hasFP16())) {
1454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
1455 }
1456 return Register();
1457}
1458
1459Register fastEmit_ISD_LRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1460switch (RetVT.SimpleTy) {
1461 case MVT::v8i16: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i16_r(Op0);
1462 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i32_r(Op0);
1463 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i64_r(Op0);
1464 default: return Register();
1465}
1466}
1467
1468Register fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i16_r(Register Op0) {
1469 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1470 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
1471 }
1472 return Register();
1473}
1474
1475Register fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i32_r(Register Op0) {
1476 if ((Subtarget->hasFP16())) {
1477 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
1478 }
1479 return Register();
1480}
1481
1482Register fastEmit_ISD_LRINT_MVT_v16f16_r(MVT RetVT, Register Op0) {
1483switch (RetVT.SimpleTy) {
1484 case MVT::v16i16: return fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i16_r(Op0);
1485 case MVT::v16i32: return fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i32_r(Op0);
1486 default: return Register();
1487}
1488}
1489
1490Register fastEmit_ISD_LRINT_MVT_v32f16_r(MVT RetVT, Register Op0) {
1491 if (RetVT.SimpleTy != MVT::v32i16)
1492 return Register();
1493 if ((Subtarget->hasFP16())) {
1494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
1495 }
1496 return Register();
1497}
1498
1499Register fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Register Op0) {
1500 if ((Subtarget->hasVLX())) {
1501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
1502 }
1503 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1504 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1505 }
1506 if ((Subtarget->hasAVX())) {
1507 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1508 }
1509 return Register();
1510}
1511
1512Register fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Register Op0) {
1513 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1514 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1515 }
1516 return Register();
1517}
1518
1519Register fastEmit_ISD_LRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1520switch (RetVT.SimpleTy) {
1521 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Op0);
1522 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Op0);
1523 default: return Register();
1524}
1525}
1526
1527Register fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Register Op0) {
1528 if ((Subtarget->hasVLX())) {
1529 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1530 }
1531 if ((Subtarget->hasAVX())) {
1532 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
1533 }
1534 return Register();
1535}
1536
1537Register fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Register Op0) {
1538 if ((Subtarget->hasDQI())) {
1539 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1540 }
1541 return Register();
1542}
1543
1544Register fastEmit_ISD_LRINT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1545switch (RetVT.SimpleTy) {
1546 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Op0);
1547 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Op0);
1548 default: return Register();
1549}
1550}
1551
1552Register fastEmit_ISD_LRINT_MVT_v16f32_r(MVT RetVT, Register Op0) {
1553 if (RetVT.SimpleTy != MVT::v16i32)
1554 return Register();
1555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
1556}
1557
1558Register fastEmit_ISD_LRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1559 if (RetVT.SimpleTy != MVT::v2i64)
1560 return Register();
1561 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1562 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1563 }
1564 return Register();
1565}
1566
1567Register fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Register Op0) {
1568 if ((Subtarget->hasVLX())) {
1569 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
1570 }
1571 if ((Subtarget->hasAVX())) {
1572 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
1573 }
1574 return Register();
1575}
1576
1577Register fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Register Op0) {
1578 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1579 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1580 }
1581 return Register();
1582}
1583
1584Register fastEmit_ISD_LRINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1585switch (RetVT.SimpleTy) {
1586 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Op0);
1587 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Op0);
1588 default: return Register();
1589}
1590}
1591
1592Register fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Register Op0) {
1593 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
1594}
1595
1596Register fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Register Op0) {
1597 if ((Subtarget->hasDQI())) {
1598 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1599 }
1600 return Register();
1601}
1602
1603Register fastEmit_ISD_LRINT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1604switch (RetVT.SimpleTy) {
1605 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Op0);
1606 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Op0);
1607 default: return Register();
1608}
1609}
1610
1611Register fastEmit_ISD_LRINT_r(MVT VT, MVT RetVT, Register Op0) {
1612 switch (VT.SimpleTy) {
1613 case MVT::f16: return fastEmit_ISD_LRINT_MVT_f16_r(RetVT, Op0);
1614 case MVT::f32: return fastEmit_ISD_LRINT_MVT_f32_r(RetVT, Op0);
1615 case MVT::f64: return fastEmit_ISD_LRINT_MVT_f64_r(RetVT, Op0);
1616 case MVT::v8f16: return fastEmit_ISD_LRINT_MVT_v8f16_r(RetVT, Op0);
1617 case MVT::v16f16: return fastEmit_ISD_LRINT_MVT_v16f16_r(RetVT, Op0);
1618 case MVT::v32f16: return fastEmit_ISD_LRINT_MVT_v32f16_r(RetVT, Op0);
1619 case MVT::v4f32: return fastEmit_ISD_LRINT_MVT_v4f32_r(RetVT, Op0);
1620 case MVT::v8f32: return fastEmit_ISD_LRINT_MVT_v8f32_r(RetVT, Op0);
1621 case MVT::v16f32: return fastEmit_ISD_LRINT_MVT_v16f32_r(RetVT, Op0);
1622 case MVT::v2f64: return fastEmit_ISD_LRINT_MVT_v2f64_r(RetVT, Op0);
1623 case MVT::v4f64: return fastEmit_ISD_LRINT_MVT_v4f64_r(RetVT, Op0);
1624 case MVT::v8f64: return fastEmit_ISD_LRINT_MVT_v8f64_r(RetVT, Op0);
1625 default: return Register();
1626 }
1627}
1628
1629// FastEmit functions for ISD::SCALAR_TO_VECTOR.
1630
1631Register fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
1632 if (RetVT.SimpleTy != MVT::v4i32)
1633 return Register();
1634 if ((Subtarget->hasAVX512())) {
1635 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIZrr, RC: &X86::VR128XRegClass, Op0);
1636 }
1637 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1638 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1639 }
1640 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1641 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1642 }
1643 return Register();
1644}
1645
1646Register fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
1647 if (RetVT.SimpleTy != MVT::v2i64)
1648 return Register();
1649 if ((Subtarget->hasAVX512())) {
1650 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIZrr, RC: &X86::VR128XRegClass, Op0);
1651 }
1652 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1653 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1654 }
1655 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1656 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1657 }
1658 return Register();
1659}
1660
1661Register fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
1662 switch (VT.SimpleTy) {
1663 case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0);
1664 case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0);
1665 default: return Register();
1666 }
1667}
1668
1669// FastEmit functions for ISD::SIGN_EXTEND.
1670
1671Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Register Op0) {
1672 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr8, RC: &X86::GR32RegClass, Op0);
1673}
1674
1675Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Register Op0) {
1676 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr8, RC: &X86::GR64RegClass, Op0);
1677}
1678
1679Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
1680switch (RetVT.SimpleTy) {
1681 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0);
1682 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0);
1683 default: return Register();
1684}
1685}
1686
1687Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Register Op0) {
1688 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr16, RC: &X86::GR32RegClass, Op0);
1689}
1690
1691Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Register Op0) {
1692 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr16, RC: &X86::GR64RegClass, Op0);
1693}
1694
1695Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, Register Op0) {
1696switch (RetVT.SimpleTy) {
1697 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0);
1698 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0);
1699 default: return Register();
1700}
1701}
1702
1703Register fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, Register Op0) {
1704 if (RetVT.SimpleTy != MVT::i64)
1705 return Register();
1706 if ((Subtarget->is64Bit())) {
1707 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr32, RC: &X86::GR64RegClass, Op0);
1708 }
1709 return Register();
1710}
1711
1712Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, Register Op0) {
1713 if (RetVT.SimpleTy != MVT::v2i64)
1714 return Register();
1715 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1716 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ128rk, RC: &X86::VR128XRegClass, Op0);
1717 }
1718 return Register();
1719}
1720
1721Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Register Op0) {
1722 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1723 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ128rk, RC: &X86::VR128XRegClass, Op0);
1724 }
1725 return Register();
1726}
1727
1728Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Register Op0) {
1729 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1730 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ256rk, RC: &X86::VR256XRegClass, Op0);
1731 }
1732 return Register();
1733}
1734
1735Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, Register Op0) {
1736switch (RetVT.SimpleTy) {
1737 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0);
1738 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0);
1739 default: return Register();
1740}
1741}
1742
1743Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Register Op0) {
1744 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1745 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ128rk, RC: &X86::VR128XRegClass, Op0);
1746 }
1747 return Register();
1748}
1749
1750Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Register Op0) {
1751 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1752 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ256rk, RC: &X86::VR256XRegClass, Op0);
1753 }
1754 return Register();
1755}
1756
1757Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Register Op0) {
1758 if ((Subtarget->hasDQI())) {
1759 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZrk, RC: &X86::VR512RegClass, Op0);
1760 }
1761 return Register();
1762}
1763
1764Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, Register Op0) {
1765switch (RetVT.SimpleTy) {
1766 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0);
1767 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0);
1768 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0);
1769 default: return Register();
1770}
1771}
1772
1773Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Register Op0) {
1774 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1775 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ128rk, RC: &X86::VR128XRegClass, Op0);
1776 }
1777 return Register();
1778}
1779
1780Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Register Op0) {
1781 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1782 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ256rk, RC: &X86::VR256XRegClass, Op0);
1783 }
1784 return Register();
1785}
1786
1787Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Register Op0) {
1788 if ((Subtarget->hasDQI())) {
1789 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZrk, RC: &X86::VR512RegClass, Op0);
1790 }
1791 return Register();
1792}
1793
1794Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, Register Op0) {
1795switch (RetVT.SimpleTy) {
1796 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0);
1797 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0);
1798 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0);
1799 default: return Register();
1800}
1801}
1802
1803Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Register Op0) {
1804 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1805 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ256rk, RC: &X86::VR256XRegClass, Op0);
1806 }
1807 return Register();
1808}
1809
1810Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Register Op0) {
1811 if ((Subtarget->hasBWI())) {
1812 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZrk, RC: &X86::VR512RegClass, Op0);
1813 }
1814 return Register();
1815}
1816
1817Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, Register Op0) {
1818switch (RetVT.SimpleTy) {
1819 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0);
1820 case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0);
1821 default: return Register();
1822}
1823}
1824
1825Register fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, Register Op0) {
1826 if (RetVT.SimpleTy != MVT::v64i8)
1827 return Register();
1828 if ((Subtarget->hasBWI())) {
1829 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZrk, RC: &X86::VR512RegClass, Op0);
1830 }
1831 return Register();
1832}
1833
1834Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Register Op0) {
1835 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1836 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWYrr, RC: &X86::VR256RegClass, Op0);
1837 }
1838 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1839 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
1840 }
1841 return Register();
1842}
1843
1844Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Register Op0) {
1845 if ((Subtarget->hasAVX512())) {
1846 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZrr, RC: &X86::VR512RegClass, Op0);
1847 }
1848 return Register();
1849}
1850
1851Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(MVT RetVT, Register Op0) {
1852switch (RetVT.SimpleTy) {
1853 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
1854 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
1855 default: return Register();
1856}
1857}
1858
1859Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(MVT RetVT, Register Op0) {
1860 if (RetVT.SimpleTy != MVT::v32i16)
1861 return Register();
1862 if ((Subtarget->hasBWI())) {
1863 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZrr, RC: &X86::VR512RegClass, Op0);
1864 }
1865 return Register();
1866}
1867
1868Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Register Op0) {
1869 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1870 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDYrr, RC: &X86::VR256RegClass, Op0);
1871 }
1872 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1873 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
1874 }
1875 return Register();
1876}
1877
1878Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Register Op0) {
1879 if ((Subtarget->hasAVX512())) {
1880 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZrr, RC: &X86::VR512RegClass, Op0);
1881 }
1882 return Register();
1883}
1884
1885Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(MVT RetVT, Register Op0) {
1886switch (RetVT.SimpleTy) {
1887 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
1888 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
1889 default: return Register();
1890}
1891}
1892
1893Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(MVT RetVT, Register Op0) {
1894 if (RetVT.SimpleTy != MVT::v16i32)
1895 return Register();
1896 if ((Subtarget->hasAVX512())) {
1897 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZrr, RC: &X86::VR512RegClass, Op0);
1898 }
1899 return Register();
1900}
1901
1902Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(MVT RetVT, Register Op0) {
1903 if (RetVT.SimpleTy != MVT::v4i64)
1904 return Register();
1905 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1906 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQYrr, RC: &X86::VR256RegClass, Op0);
1907 }
1908 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1909 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
1910 }
1911 return Register();
1912}
1913
1914Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(MVT RetVT, Register Op0) {
1915 if (RetVT.SimpleTy != MVT::v8i64)
1916 return Register();
1917 if ((Subtarget->hasAVX512())) {
1918 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZrr, RC: &X86::VR512RegClass, Op0);
1919 }
1920 return Register();
1921}
1922
1923Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
1924 switch (VT.SimpleTy) {
1925 case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0);
1926 case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0);
1927 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0);
1928 case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0);
1929 case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0);
1930 case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0);
1931 case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0);
1932 case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0);
1933 case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0);
1934 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(RetVT, Op0);
1935 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(RetVT, Op0);
1936 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(RetVT, Op0);
1937 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(RetVT, Op0);
1938 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(RetVT, Op0);
1939 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(RetVT, Op0);
1940 default: return Register();
1941 }
1942}
1943
1944// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
1945
1946Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1947 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1948 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1949 }
1950 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1951 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1952 }
1953 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1954 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
1955 }
1956 return Register();
1957}
1958
1959Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1960 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1961 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
1962 }
1963 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1964 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
1965 }
1966 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1967 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
1968 }
1969 return Register();
1970}
1971
1972Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Register Op0) {
1973 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1974 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDYrr, RC: &X86::VR256RegClass, Op0);
1975 }
1976 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1977 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
1978 }
1979 return Register();
1980}
1981
1982Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1983 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1984 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
1985 }
1986 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1987 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
1988 }
1989 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1990 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
1991 }
1992 return Register();
1993}
1994
1995Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Register Op0) {
1996 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1997 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQYrr, RC: &X86::VR256RegClass, Op0);
1998 }
1999 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2000 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
2001 }
2002 return Register();
2003}
2004
2005Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Register Op0) {
2006 if ((Subtarget->hasAVX512())) {
2007 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZrr, RC: &X86::VR512RegClass, Op0);
2008 }
2009 return Register();
2010}
2011
2012Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, Register Op0) {
2013switch (RetVT.SimpleTy) {
2014 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
2015 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
2016 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
2017 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
2018 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
2019 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
2020 default: return Register();
2021}
2022}
2023
2024Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Register Op0) {
2025 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2026 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
2027 }
2028 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2029 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
2030 }
2031 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2032 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
2033 }
2034 return Register();
2035}
2036
2037Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Register Op0) {
2038 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2039 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
2040 }
2041 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2042 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
2043 }
2044 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2045 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
2046 }
2047 return Register();
2048}
2049
2050Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Register Op0) {
2051 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2052 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQYrr, RC: &X86::VR256RegClass, Op0);
2053 }
2054 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2055 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
2056 }
2057 return Register();
2058}
2059
2060Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, Register Op0) {
2061switch (RetVT.SimpleTy) {
2062 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
2063 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
2064 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
2065 default: return Register();
2066}
2067}
2068
2069Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, Register Op0) {
2070 if (RetVT.SimpleTy != MVT::v2i64)
2071 return Register();
2072 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2073 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
2074 }
2075 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2076 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
2077 }
2078 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2079 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
2080 }
2081 return Register();
2082}
2083
2084Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, Register Op0) {
2085 switch (VT.SimpleTy) {
2086 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
2087 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
2088 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
2089 default: return Register();
2090 }
2091}
2092
2093// FastEmit functions for ISD::SINT_TO_FP.
2094
2095Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
2096 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2097 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2098 }
2099 return Register();
2100}
2101
2102Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
2103 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2104 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2105 }
2106 return Register();
2107}
2108
2109Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
2110switch (RetVT.SimpleTy) {
2111 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2112 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2113 default: return Register();
2114}
2115}
2116
2117Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
2118 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2119 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2120 }
2121 return Register();
2122}
2123
2124Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
2125 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2126 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2127 }
2128 return Register();
2129}
2130
2131Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
2132switch (RetVT.SimpleTy) {
2133 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2134 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2135 default: return Register();
2136}
2137}
2138
2139Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2140 if (RetVT.SimpleTy != MVT::v8f16)
2141 return Register();
2142 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2144 }
2145 return Register();
2146}
2147
2148Register fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
2149 if (RetVT.SimpleTy != MVT::v16f16)
2150 return Register();
2151 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2152 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2153 }
2154 return Register();
2155}
2156
2157Register fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
2158 if (RetVT.SimpleTy != MVT::v32f16)
2159 return Register();
2160 if ((Subtarget->hasFP16())) {
2161 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2162 }
2163 return Register();
2164}
2165
2166Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
2167 if ((Subtarget->hasVLX())) {
2168 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2169 }
2170 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2171 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2172 }
2173 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2174 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2175 }
2176 return Register();
2177}
2178
2179Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
2180 if ((Subtarget->hasVLX())) {
2181 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2182 }
2183 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2184 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2185 }
2186 return Register();
2187}
2188
2189Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2190switch (RetVT.SimpleTy) {
2191 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2192 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2193 default: return Register();
2194}
2195}
2196
2197Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
2198 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2199 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2200 }
2201 return Register();
2202}
2203
2204Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
2205 if ((Subtarget->hasVLX())) {
2206 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2207 }
2208 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2210 }
2211 return Register();
2212}
2213
2214Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
2215 if ((Subtarget->hasAVX512())) {
2216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2217 }
2218 return Register();
2219}
2220
2221Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
2222switch (RetVT.SimpleTy) {
2223 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2224 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2225 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2226 default: return Register();
2227}
2228}
2229
2230Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
2231 if ((Subtarget->hasFP16())) {
2232 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2233 }
2234 return Register();
2235}
2236
2237Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
2238 if ((Subtarget->hasAVX512())) {
2239 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2240 }
2241 return Register();
2242}
2243
2244Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
2245switch (RetVT.SimpleTy) {
2246 case MVT::v16f16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2247 case MVT::v16f32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2248 default: return Register();
2249}
2250}
2251
2252Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2253 if (RetVT.SimpleTy != MVT::v2f64)
2254 return Register();
2255 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2256 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2257 }
2258 return Register();
2259}
2260
2261Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
2262 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2263 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2264 }
2265 return Register();
2266}
2267
2268Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
2269 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2270 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2271 }
2272 return Register();
2273}
2274
2275Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
2276switch (RetVT.SimpleTy) {
2277 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2278 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2279 default: return Register();
2280}
2281}
2282
2283Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
2284 if ((Subtarget->hasFP16())) {
2285 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2286 }
2287 return Register();
2288}
2289
2290Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
2291 if ((Subtarget->hasDQI())) {
2292 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2293 }
2294 return Register();
2295}
2296
2297Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
2298 if ((Subtarget->hasDQI())) {
2299 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2300 }
2301 return Register();
2302}
2303
2304Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
2305switch (RetVT.SimpleTy) {
2306 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
2307 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
2308 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
2309 default: return Register();
2310}
2311}
2312
2313Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
2314 switch (VT.SimpleTy) {
2315 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
2316 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
2317 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2318 case MVT::v16i16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
2319 case MVT::v32i16: return fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
2320 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2321 case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
2322 case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
2323 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
2324 case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
2325 case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
2326 default: return Register();
2327 }
2328}
2329
2330// FastEmit functions for ISD::STRICT_FP_EXTEND.
2331
2332Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2333 if (RetVT.SimpleTy != MVT::f64)
2334 return Register();
2335 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2336 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
2337 }
2338 return Register();
2339}
2340
2341Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Register Op0) {
2342 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2343 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
2344 }
2345 return Register();
2346}
2347
2348Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Register Op0) {
2349 if ((Subtarget->hasFP16())) {
2350 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
2351 }
2352 return Register();
2353}
2354
2355Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2356switch (RetVT.SimpleTy) {
2357 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
2358 case MVT::v8f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
2359 default: return Register();
2360}
2361}
2362
2363Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(MVT RetVT, Register Op0) {
2364 if (RetVT.SimpleTy != MVT::v16f32)
2365 return Register();
2366 if ((Subtarget->hasFP16())) {
2367 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
2368 }
2369 return Register();
2370}
2371
2372Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2373 if (RetVT.SimpleTy != MVT::v4f64)
2374 return Register();
2375 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2376 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2377 }
2378 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2379 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
2380 }
2381 return Register();
2382}
2383
2384Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(MVT RetVT, Register Op0) {
2385 if (RetVT.SimpleTy != MVT::v8f64)
2386 return Register();
2387 if ((Subtarget->hasAVX512())) {
2388 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
2389 }
2390 return Register();
2391}
2392
2393Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2394 switch (VT.SimpleTy) {
2395 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2396 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
2397 case MVT::v16f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
2398 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
2399 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
2400 default: return Register();
2401 }
2402}
2403
2404// FastEmit functions for ISD::STRICT_FP_ROUND.
2405
2406Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2407 if (RetVT.SimpleTy != MVT::f32)
2408 return Register();
2409 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2410 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
2411 }
2412 return Register();
2413}
2414
2415Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2416 switch (VT.SimpleTy) {
2417 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
2418 default: return Register();
2419 }
2420}
2421
2422// FastEmit functions for ISD::STRICT_FP_TO_SINT.
2423
2424Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
2425 if ((Subtarget->hasFP16())) {
2426 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
2427 }
2428 return Register();
2429}
2430
2431Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
2432 if ((Subtarget->hasFP16())) {
2433 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2434 }
2435 return Register();
2436}
2437
2438Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
2439switch (RetVT.SimpleTy) {
2440 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
2441 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
2442 default: return Register();
2443}
2444}
2445
2446Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
2447 if ((Subtarget->hasAVX512())) {
2448 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
2449 }
2450 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2451 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2452 }
2453 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2455 }
2456 return Register();
2457}
2458
2459Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
2460 if ((Subtarget->hasAVX512())) {
2461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2462 }
2463 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2464 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2465 }
2466 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2467 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2468 }
2469 return Register();
2470}
2471
2472Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
2473switch (RetVT.SimpleTy) {
2474 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
2475 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
2476 default: return Register();
2477}
2478}
2479
2480Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
2481 if ((Subtarget->hasAVX512())) {
2482 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
2483 }
2484 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2485 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2486 }
2487 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2488 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2489 }
2490 return Register();
2491}
2492
2493Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
2494 if ((Subtarget->hasAVX512())) {
2495 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2496 }
2497 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2498 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2499 }
2500 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2502 }
2503 return Register();
2504}
2505
2506Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
2507switch (RetVT.SimpleTy) {
2508 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
2509 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
2510 default: return Register();
2511}
2512}
2513
2514Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
2515 if (RetVT.SimpleTy != MVT::v4i32)
2516 return Register();
2517 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2518 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
2519 }
2520 return Register();
2521}
2522
2523Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2524 switch (VT.SimpleTy) {
2525 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
2526 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
2527 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
2528 case MVT::v4f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
2529 default: return Register();
2530 }
2531}
2532
2533// FastEmit functions for ISD::STRICT_FP_TO_UINT.
2534
2535Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
2536 if ((Subtarget->hasFP16())) {
2537 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
2538 }
2539 return Register();
2540}
2541
2542Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
2543 if ((Subtarget->hasFP16())) {
2544 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2545 }
2546 return Register();
2547}
2548
2549Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
2550switch (RetVT.SimpleTy) {
2551 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
2552 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
2553 default: return Register();
2554}
2555}
2556
2557Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
2558 if ((Subtarget->hasAVX512())) {
2559 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
2560 }
2561 return Register();
2562}
2563
2564Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
2565 if ((Subtarget->hasAVX512())) {
2566 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2567 }
2568 return Register();
2569}
2570
2571Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
2572switch (RetVT.SimpleTy) {
2573 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
2574 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
2575 default: return Register();
2576}
2577}
2578
2579Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
2580 if ((Subtarget->hasAVX512())) {
2581 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
2582 }
2583 return Register();
2584}
2585
2586Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
2587 if ((Subtarget->hasAVX512())) {
2588 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2589 }
2590 return Register();
2591}
2592
2593Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
2594switch (RetVT.SimpleTy) {
2595 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
2596 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
2597 default: return Register();
2598}
2599}
2600
2601Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2602 switch (VT.SimpleTy) {
2603 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
2604 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
2605 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
2606 default: return Register();
2607 }
2608}
2609
2610// FastEmit functions for ISD::STRICT_FSQRT.
2611
2612Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
2613 if (RetVT.SimpleTy != MVT::f32)
2614 return Register();
2615 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2616 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
2617 }
2618 if ((!Subtarget->hasSSE1())) {
2619 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
2620 }
2621 return Register();
2622}
2623
2624Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
2625 if (RetVT.SimpleTy != MVT::f64)
2626 return Register();
2627 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2628 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
2629 }
2630 if ((!Subtarget->hasSSE2())) {
2631 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
2632 }
2633 return Register();
2634}
2635
2636Register fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(MVT RetVT, Register Op0) {
2637 if (RetVT.SimpleTy != MVT::f80)
2638 return Register();
2639 if ((Subtarget->hasX87())) {
2640 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
2641 }
2642 return Register();
2643}
2644
2645Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2646 if (RetVT.SimpleTy != MVT::v8f16)
2647 return Register();
2648 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2649 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
2650 }
2651 return Register();
2652}
2653
2654Register fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(MVT RetVT, Register Op0) {
2655 if (RetVT.SimpleTy != MVT::v16f16)
2656 return Register();
2657 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2658 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
2659 }
2660 return Register();
2661}
2662
2663Register fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(MVT RetVT, Register Op0) {
2664 if (RetVT.SimpleTy != MVT::v32f16)
2665 return Register();
2666 if ((Subtarget->hasFP16())) {
2667 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
2668 }
2669 return Register();
2670}
2671
2672Register fastEmit_ISD_STRICT_FSQRT_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2673 if (RetVT.SimpleTy != MVT::v8bf16)
2674 return Register();
2675 if ((Subtarget->hasAVX10_2())) {
2676 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
2677 }
2678 return Register();
2679}
2680
2681Register fastEmit_ISD_STRICT_FSQRT_MVT_v16bf16_r(MVT RetVT, Register Op0) {
2682 if (RetVT.SimpleTy != MVT::v16bf16)
2683 return Register();
2684 if ((Subtarget->hasAVX10_2())) {
2685 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
2686 }
2687 return Register();
2688}
2689
2690Register fastEmit_ISD_STRICT_FSQRT_MVT_v32bf16_r(MVT RetVT, Register Op0) {
2691 if (RetVT.SimpleTy != MVT::v32bf16)
2692 return Register();
2693 if ((Subtarget->hasAVX10_2())) {
2694 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
2695 }
2696 return Register();
2697}
2698
2699Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2700 if (RetVT.SimpleTy != MVT::v4f32)
2701 return Register();
2702 if ((Subtarget->hasVLX())) {
2703 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
2704 }
2705 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2706 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
2707 }
2708 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2709 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
2710 }
2711 return Register();
2712}
2713
2714Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
2715 if (RetVT.SimpleTy != MVT::v8f32)
2716 return Register();
2717 if ((Subtarget->hasVLX())) {
2718 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
2719 }
2720 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2721 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
2722 }
2723 return Register();
2724}
2725
2726Register fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(MVT RetVT, Register Op0) {
2727 if (RetVT.SimpleTy != MVT::v16f32)
2728 return Register();
2729 if ((Subtarget->hasAVX512())) {
2730 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
2731 }
2732 return Register();
2733}
2734
2735Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2736 if (RetVT.SimpleTy != MVT::v2f64)
2737 return Register();
2738 if ((Subtarget->hasVLX())) {
2739 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
2740 }
2741 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2742 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
2743 }
2744 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2745 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
2746 }
2747 return Register();
2748}
2749
2750Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(MVT RetVT, Register Op0) {
2751 if (RetVT.SimpleTy != MVT::v4f64)
2752 return Register();
2753 if ((Subtarget->hasVLX())) {
2754 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
2755 }
2756 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2757 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
2758 }
2759 return Register();
2760}
2761
2762Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(MVT RetVT, Register Op0) {
2763 if (RetVT.SimpleTy != MVT::v8f64)
2764 return Register();
2765 if ((Subtarget->hasAVX512())) {
2766 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
2767 }
2768 return Register();
2769}
2770
2771Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
2772 switch (VT.SimpleTy) {
2773 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
2774 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
2775 case MVT::f80: return fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(RetVT, Op0);
2776 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
2777 case MVT::v16f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(RetVT, Op0);
2778 case MVT::v32f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(RetVT, Op0);
2779 case MVT::v8bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8bf16_r(RetVT, Op0);
2780 case MVT::v16bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16bf16_r(RetVT, Op0);
2781 case MVT::v32bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32bf16_r(RetVT, Op0);
2782 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
2783 case MVT::v8f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(RetVT, Op0);
2784 case MVT::v16f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(RetVT, Op0);
2785 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
2786 case MVT::v4f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(RetVT, Op0);
2787 case MVT::v8f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(RetVT, Op0);
2788 default: return Register();
2789 }
2790}
2791
2792// FastEmit functions for ISD::STRICT_SINT_TO_FP.
2793
2794Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
2795 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2796 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2797 }
2798 return Register();
2799}
2800
2801Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
2802 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2803 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2804 }
2805 return Register();
2806}
2807
2808Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
2809switch (RetVT.SimpleTy) {
2810 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2811 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2812 default: return Register();
2813}
2814}
2815
2816Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
2817 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2818 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2819 }
2820 return Register();
2821}
2822
2823Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
2824 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2825 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2826 }
2827 return Register();
2828}
2829
2830Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
2831switch (RetVT.SimpleTy) {
2832 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2833 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2834 default: return Register();
2835}
2836}
2837
2838Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2839 if (RetVT.SimpleTy != MVT::v8f16)
2840 return Register();
2841 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2842 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2843 }
2844 return Register();
2845}
2846
2847Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
2848 if (RetVT.SimpleTy != MVT::v16f16)
2849 return Register();
2850 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2851 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2852 }
2853 return Register();
2854}
2855
2856Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
2857 if (RetVT.SimpleTy != MVT::v32f16)
2858 return Register();
2859 if ((Subtarget->hasFP16())) {
2860 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2861 }
2862 return Register();
2863}
2864
2865Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
2866 if ((Subtarget->hasVLX())) {
2867 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2868 }
2869 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2870 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2871 }
2872 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2873 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2874 }
2875 return Register();
2876}
2877
2878Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
2879 if ((Subtarget->hasVLX())) {
2880 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2881 }
2882 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2883 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2884 }
2885 return Register();
2886}
2887
2888Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2889switch (RetVT.SimpleTy) {
2890 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2891 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2892 default: return Register();
2893}
2894}
2895
2896Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
2897 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2898 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2899 }
2900 return Register();
2901}
2902
2903Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
2904 if ((Subtarget->hasVLX())) {
2905 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2906 }
2907 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2908 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2909 }
2910 return Register();
2911}
2912
2913Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
2914 if ((Subtarget->hasAVX512())) {
2915 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2916 }
2917 return Register();
2918}
2919
2920Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
2921switch (RetVT.SimpleTy) {
2922 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2923 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2924 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2925 default: return Register();
2926}
2927}
2928
2929Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
2930 if ((Subtarget->hasFP16())) {
2931 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2932 }
2933 return Register();
2934}
2935
2936Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
2937 if ((Subtarget->hasAVX512())) {
2938 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2939 }
2940 return Register();
2941}
2942
2943Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
2944switch (RetVT.SimpleTy) {
2945 case MVT::v16f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2946 case MVT::v16f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2947 default: return Register();
2948}
2949}
2950
2951Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2952 if (RetVT.SimpleTy != MVT::v2f64)
2953 return Register();
2954 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2955 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2956 }
2957 return Register();
2958}
2959
2960Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
2961 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2963 }
2964 return Register();
2965}
2966
2967Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
2968 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2969 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2970 }
2971 return Register();
2972}
2973
2974Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
2975switch (RetVT.SimpleTy) {
2976 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2977 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2978 default: return Register();
2979}
2980}
2981
2982Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
2983 if ((Subtarget->hasFP16())) {
2984 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2985 }
2986 return Register();
2987}
2988
2989Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
2990 if ((Subtarget->hasDQI())) {
2991 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2992 }
2993 return Register();
2994}
2995
2996Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
2997 if ((Subtarget->hasDQI())) {
2998 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2999 }
3000 return Register();
3001}
3002
3003Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3004switch (RetVT.SimpleTy) {
3005 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3006 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3007 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3008 default: return Register();
3009}
3010}
3011
3012Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3013 switch (VT.SimpleTy) {
3014 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
3015 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
3016 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3017 case MVT::v16i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3018 case MVT::v32i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3019 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3020 case MVT::v8i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3021 case MVT::v16i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3022 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3023 case MVT::v4i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3024 case MVT::v8i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3025 default: return Register();
3026 }
3027}
3028
3029// FastEmit functions for ISD::STRICT_UINT_TO_FP.
3030
3031Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3032 if (RetVT.SimpleTy != MVT::v8f16)
3033 return Register();
3034 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3035 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3036 }
3037 return Register();
3038}
3039
3040Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
3041 if (RetVT.SimpleTy != MVT::v16f16)
3042 return Register();
3043 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3044 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3045 }
3046 return Register();
3047}
3048
3049Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
3050 if (RetVT.SimpleTy != MVT::v32f16)
3051 return Register();
3052 if ((Subtarget->hasFP16())) {
3053 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3054 }
3055 return Register();
3056}
3057
3058Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
3059 if ((Subtarget->hasVLX())) {
3060 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3061 }
3062 return Register();
3063}
3064
3065Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
3066 if ((Subtarget->hasVLX())) {
3067 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3068 }
3069 return Register();
3070}
3071
3072Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3073switch (RetVT.SimpleTy) {
3074 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3075 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3076 default: return Register();
3077}
3078}
3079
3080Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
3081 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3082 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3083 }
3084 return Register();
3085}
3086
3087Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
3088 if ((Subtarget->hasVLX())) {
3089 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3090 }
3091 return Register();
3092}
3093
3094Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
3095 if ((Subtarget->hasAVX512())) {
3096 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3097 }
3098 return Register();
3099}
3100
3101Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
3102switch (RetVT.SimpleTy) {
3103 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3104 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3105 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3106 default: return Register();
3107}
3108}
3109
3110Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
3111 if ((Subtarget->hasFP16())) {
3112 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3113 }
3114 return Register();
3115}
3116
3117Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
3118 if ((Subtarget->hasAVX512())) {
3119 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3120 }
3121 return Register();
3122}
3123
3124Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
3125switch (RetVT.SimpleTy) {
3126 case MVT::v16f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3127 case MVT::v16f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3128 default: return Register();
3129}
3130}
3131
3132Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
3133 if (RetVT.SimpleTy != MVT::v2f64)
3134 return Register();
3135 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3136 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3137 }
3138 return Register();
3139}
3140
3141Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
3142 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3144 }
3145 return Register();
3146}
3147
3148Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3149 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3150 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3151 }
3152 return Register();
3153}
3154
3155Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3156switch (RetVT.SimpleTy) {
3157 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3158 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3159 default: return Register();
3160}
3161}
3162
3163Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3164 if ((Subtarget->hasFP16())) {
3165 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3166 }
3167 return Register();
3168}
3169
3170Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3171 if ((Subtarget->hasDQI())) {
3172 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3173 }
3174 return Register();
3175}
3176
3177Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3178 if ((Subtarget->hasDQI())) {
3179 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3180 }
3181 return Register();
3182}
3183
3184Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3185switch (RetVT.SimpleTy) {
3186 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3187 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3188 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3189 default: return Register();
3190}
3191}
3192
3193Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3194 switch (VT.SimpleTy) {
3195 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3196 case MVT::v16i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3197 case MVT::v32i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3198 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3199 case MVT::v8i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3200 case MVT::v16i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3201 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3202 case MVT::v4i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3203 case MVT::v8i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3204 default: return Register();
3205 }
3206}
3207
3208// FastEmit functions for ISD::TRUNCATE.
3209
3210Register fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, Register Op0) {
3211 if (RetVT.SimpleTy != MVT::i8)
3212 return Register();
3213 if ((Subtarget->is64Bit())) {
3214 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: X86::sub_8bit);
3215 }
3216 return Register();
3217}
3218
3219Register fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Register Op0) {
3220 if ((Subtarget->is64Bit())) {
3221 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3222 }
3223 return Register();
3224}
3225
3226Register fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Register Op0) {
3227 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3228}
3229
3230Register fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, Register Op0) {
3231switch (RetVT.SimpleTy) {
3232 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0);
3233 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0);
3234 default: return Register();
3235}
3236}
3237
3238Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Register Op0) {
3239 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3240}
3241
3242Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Register Op0) {
3243 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3244}
3245
3246Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Register Op0) {
3247 return fastEmitInst_extractsubreg(RetVT: MVT::i32, Op0, Idx: X86::sub_32bit);
3248}
3249
3250Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
3251switch (RetVT.SimpleTy) {
3252 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0);
3253 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0);
3254 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0);
3255 default: return Register();
3256}
3257}
3258
3259Register fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, Register Op0) {
3260 if (RetVT.SimpleTy != MVT::v16i8)
3261 return Register();
3262 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3263 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ256rr, RC: &X86::VR128XRegClass, Op0);
3264 }
3265 return Register();
3266}
3267
3268Register fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, Register Op0) {
3269 if (RetVT.SimpleTy != MVT::v32i8)
3270 return Register();
3271 if ((Subtarget->hasBWI())) {
3272 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZrr, RC: &X86::VR256XRegClass, Op0);
3273 }
3274 return Register();
3275}
3276
3277Register fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, Register Op0) {
3278 if (RetVT.SimpleTy != MVT::v8i16)
3279 return Register();
3280 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3281 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ256rr, RC: &X86::VR128XRegClass, Op0);
3282 }
3283 return Register();
3284}
3285
3286Register fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Register Op0) {
3287 if ((Subtarget->hasAVX512())) {
3288 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZrr, RC: &X86::VR128XRegClass, Op0);
3289 }
3290 return Register();
3291}
3292
3293Register fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Register Op0) {
3294 if ((Subtarget->hasAVX512())) {
3295 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZrr, RC: &X86::VR256XRegClass, Op0);
3296 }
3297 return Register();
3298}
3299
3300Register fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, Register Op0) {
3301switch (RetVT.SimpleTy) {
3302 case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0);
3303 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0);
3304 default: return Register();
3305}
3306}
3307
3308Register fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, Register Op0) {
3309 if (RetVT.SimpleTy != MVT::v4i32)
3310 return Register();
3311 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3312 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ256rr, RC: &X86::VR128XRegClass, Op0);
3313 }
3314 return Register();
3315}
3316
3317Register fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Register Op0) {
3318 if ((Subtarget->hasAVX512())) {
3319 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZrr, RC: &X86::VR128XRegClass, Op0);
3320 }
3321 return Register();
3322}
3323
3324Register fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Register Op0) {
3325 if ((Subtarget->hasAVX512())) {
3326 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZrr, RC: &X86::VR256XRegClass, Op0);
3327 }
3328 return Register();
3329}
3330
3331Register fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, Register Op0) {
3332switch (RetVT.SimpleTy) {
3333 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0);
3334 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0);
3335 default: return Register();
3336}
3337}
3338
3339Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3340 switch (VT.SimpleTy) {
3341 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0);
3342 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0);
3343 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
3344 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0);
3345 case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0);
3346 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0);
3347 case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0);
3348 case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0);
3349 case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0);
3350 default: return Register();
3351 }
3352}
3353
3354// FastEmit functions for ISD::UINT_TO_FP.
3355
3356Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3357 if (RetVT.SimpleTy != MVT::v8f16)
3358 return Register();
3359 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3360 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3361 }
3362 return Register();
3363}
3364
3365Register fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
3366 if (RetVT.SimpleTy != MVT::v16f16)
3367 return Register();
3368 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3369 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3370 }
3371 return Register();
3372}
3373
3374Register fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
3375 if (RetVT.SimpleTy != MVT::v32f16)
3376 return Register();
3377 if ((Subtarget->hasFP16())) {
3378 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3379 }
3380 return Register();
3381}
3382
3383Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
3384 if ((Subtarget->hasVLX())) {
3385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3386 }
3387 return Register();
3388}
3389
3390Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
3391 if ((Subtarget->hasVLX())) {
3392 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3393 }
3394 return Register();
3395}
3396
3397Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3398switch (RetVT.SimpleTy) {
3399 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3400 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3401 default: return Register();
3402}
3403}
3404
3405Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
3406 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3407 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3408 }
3409 return Register();
3410}
3411
3412Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
3413 if ((Subtarget->hasVLX())) {
3414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3415 }
3416 return Register();
3417}
3418
3419Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
3420 if ((Subtarget->hasAVX512())) {
3421 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3422 }
3423 return Register();
3424}
3425
3426Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
3427switch (RetVT.SimpleTy) {
3428 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3429 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3430 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3431 default: return Register();
3432}
3433}
3434
3435Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
3436 if ((Subtarget->hasFP16())) {
3437 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3438 }
3439 return Register();
3440}
3441
3442Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
3443 if ((Subtarget->hasAVX512())) {
3444 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3445 }
3446 return Register();
3447}
3448
3449Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
3450switch (RetVT.SimpleTy) {
3451 case MVT::v16f16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3452 case MVT::v16f32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3453 default: return Register();
3454}
3455}
3456
3457Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
3458 if (RetVT.SimpleTy != MVT::v2f64)
3459 return Register();
3460 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3462 }
3463 return Register();
3464}
3465
3466Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
3467 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3469 }
3470 return Register();
3471}
3472
3473Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3474 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3475 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3476 }
3477 return Register();
3478}
3479
3480Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3481switch (RetVT.SimpleTy) {
3482 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3483 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3484 default: return Register();
3485}
3486}
3487
3488Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3489 if ((Subtarget->hasFP16())) {
3490 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3491 }
3492 return Register();
3493}
3494
3495Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3496 if ((Subtarget->hasDQI())) {
3497 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3498 }
3499 return Register();
3500}
3501
3502Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3503 if ((Subtarget->hasDQI())) {
3504 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3505 }
3506 return Register();
3507}
3508
3509Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3510switch (RetVT.SimpleTy) {
3511 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3512 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3513 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3514 default: return Register();
3515}
3516}
3517
3518Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3519 switch (VT.SimpleTy) {
3520 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3521 case MVT::v16i16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3522 case MVT::v32i16: return fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3523 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3524 case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3525 case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3526 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3527 case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3528 case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3529 default: return Register();
3530 }
3531}
3532
3533// FastEmit functions for ISD::ZERO_EXTEND.
3534
3535Register fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
3536 if (RetVT.SimpleTy != MVT::i32)
3537 return Register();
3538 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
3539}
3540
3541Register fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, Register Op0) {
3542 if (RetVT.SimpleTy != MVT::i32)
3543 return Register();
3544 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr16, RC: &X86::GR32RegClass, Op0);
3545}
3546
3547Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Register Op0) {
3548 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3549 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWYrr, RC: &X86::VR256RegClass, Op0);
3550 }
3551 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3552 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
3553 }
3554 return Register();
3555}
3556
3557Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Register Op0) {
3558 if ((Subtarget->hasAVX512())) {
3559 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZrr, RC: &X86::VR512RegClass, Op0);
3560 }
3561 return Register();
3562}
3563
3564Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(MVT RetVT, Register Op0) {
3565switch (RetVT.SimpleTy) {
3566 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
3567 case MVT::v16i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
3568 default: return Register();
3569}
3570}
3571
3572Register fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(MVT RetVT, Register Op0) {
3573 if (RetVT.SimpleTy != MVT::v32i16)
3574 return Register();
3575 if ((Subtarget->hasBWI())) {
3576 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZrr, RC: &X86::VR512RegClass, Op0);
3577 }
3578 return Register();
3579}
3580
3581Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Register Op0) {
3582 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3583 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDYrr, RC: &X86::VR256RegClass, Op0);
3584 }
3585 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3586 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
3587 }
3588 return Register();
3589}
3590
3591Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Register Op0) {
3592 if ((Subtarget->hasAVX512())) {
3593 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZrr, RC: &X86::VR512RegClass, Op0);
3594 }
3595 return Register();
3596}
3597
3598Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(MVT RetVT, Register Op0) {
3599switch (RetVT.SimpleTy) {
3600 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
3601 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
3602 default: return Register();
3603}
3604}
3605
3606Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(MVT RetVT, Register Op0) {
3607 if (RetVT.SimpleTy != MVT::v16i32)
3608 return Register();
3609 if ((Subtarget->hasAVX512())) {
3610 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZrr, RC: &X86::VR512RegClass, Op0);
3611 }
3612 return Register();
3613}
3614
3615Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(MVT RetVT, Register Op0) {
3616 if (RetVT.SimpleTy != MVT::v4i64)
3617 return Register();
3618 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3619 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQYrr, RC: &X86::VR256RegClass, Op0);
3620 }
3621 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3622 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
3623 }
3624 return Register();
3625}
3626
3627Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(MVT RetVT, Register Op0) {
3628 if (RetVT.SimpleTy != MVT::v8i64)
3629 return Register();
3630 if ((Subtarget->hasAVX512())) {
3631 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZrr, RC: &X86::VR512RegClass, Op0);
3632 }
3633 return Register();
3634}
3635
3636Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3637 switch (VT.SimpleTy) {
3638 case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0);
3639 case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0);
3640 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(RetVT, Op0);
3641 case MVT::v32i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(RetVT, Op0);
3642 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(RetVT, Op0);
3643 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(RetVT, Op0);
3644 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(RetVT, Op0);
3645 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(RetVT, Op0);
3646 default: return Register();
3647 }
3648}
3649
3650// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
3651
3652Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Register Op0) {
3653 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3654 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3655 }
3656 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3657 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3658 }
3659 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3660 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
3661 }
3662 return Register();
3663}
3664
3665Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Register Op0) {
3666 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3667 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3668 }
3669 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3670 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3671 }
3672 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3673 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
3674 }
3675 return Register();
3676}
3677
3678Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Register Op0) {
3679 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3680 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDYrr, RC: &X86::VR256RegClass, Op0);
3681 }
3682 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3683 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
3684 }
3685 return Register();
3686}
3687
3688Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Register Op0) {
3689 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3690 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3691 }
3692 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3693 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3694 }
3695 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3696 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
3697 }
3698 return Register();
3699}
3700
3701Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Register Op0) {
3702 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3703 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQYrr, RC: &X86::VR256RegClass, Op0);
3704 }
3705 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3706 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
3707 }
3708 return Register();
3709}
3710
3711Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Register Op0) {
3712 if ((Subtarget->hasAVX512())) {
3713 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZrr, RC: &X86::VR512RegClass, Op0);
3714 }
3715 return Register();
3716}
3717
3718Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, Register Op0) {
3719switch (RetVT.SimpleTy) {
3720 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
3721 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
3722 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
3723 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
3724 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
3725 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
3726 default: return Register();
3727}
3728}
3729
3730Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Register Op0) {
3731 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3732 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3733 }
3734 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3735 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3736 }
3737 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3738 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
3739 }
3740 return Register();
3741}
3742
3743Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Register Op0) {
3744 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3745 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3746 }
3747 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3748 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3749 }
3750 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3751 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
3752 }
3753 return Register();
3754}
3755
3756Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Register Op0) {
3757 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3758 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQYrr, RC: &X86::VR256RegClass, Op0);
3759 }
3760 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3761 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
3762 }
3763 return Register();
3764}
3765
3766Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, Register Op0) {
3767switch (RetVT.SimpleTy) {
3768 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
3769 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
3770 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
3771 default: return Register();
3772}
3773}
3774
3775Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, Register Op0) {
3776 if (RetVT.SimpleTy != MVT::v2i64)
3777 return Register();
3778 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3779 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3780 }
3781 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3782 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3783 }
3784 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3785 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
3786 }
3787 return Register();
3788}
3789
3790Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, Register Op0) {
3791 switch (VT.SimpleTy) {
3792 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
3793 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
3794 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
3795 default: return Register();
3796 }
3797}
3798
3799// FastEmit functions for X86ISD::CONFLICT.
3800
3801Register fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, Register Op0) {
3802 if (RetVT.SimpleTy != MVT::v4i32)
3803 return Register();
3804 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3805 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ128rr, RC: &X86::VR128XRegClass, Op0);
3806 }
3807 return Register();
3808}
3809
3810Register fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, Register Op0) {
3811 if (RetVT.SimpleTy != MVT::v8i32)
3812 return Register();
3813 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3814 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ256rr, RC: &X86::VR256XRegClass, Op0);
3815 }
3816 return Register();
3817}
3818
3819Register fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, Register Op0) {
3820 if (RetVT.SimpleTy != MVT::v16i32)
3821 return Register();
3822 if ((Subtarget->hasCDI())) {
3823 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZrr, RC: &X86::VR512RegClass, Op0);
3824 }
3825 return Register();
3826}
3827
3828Register fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, Register Op0) {
3829 if (RetVT.SimpleTy != MVT::v2i64)
3830 return Register();
3831 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3832 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ128rr, RC: &X86::VR128XRegClass, Op0);
3833 }
3834 return Register();
3835}
3836
3837Register fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, Register Op0) {
3838 if (RetVT.SimpleTy != MVT::v4i64)
3839 return Register();
3840 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3841 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ256rr, RC: &X86::VR256XRegClass, Op0);
3842 }
3843 return Register();
3844}
3845
3846Register fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, Register Op0) {
3847 if (RetVT.SimpleTy != MVT::v8i64)
3848 return Register();
3849 if ((Subtarget->hasCDI())) {
3850 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZrr, RC: &X86::VR512RegClass, Op0);
3851 }
3852 return Register();
3853}
3854
3855Register fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, Register Op0) {
3856 switch (VT.SimpleTy) {
3857 case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0);
3858 case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0);
3859 case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0);
3860 case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0);
3861 case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0);
3862 case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0);
3863 default: return Register();
3864 }
3865}
3866
3867// FastEmit functions for X86ISD::CVTNEPS2BF16.
3868
3869Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(MVT RetVT, Register Op0) {
3870 if (RetVT.SimpleTy != MVT::v8bf16)
3871 return Register();
3872 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3873 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0);
3874 }
3875 if ((Subtarget->hasAVXNECONVERT())) {
3876 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16rr, RC: &X86::VR128RegClass, Op0);
3877 }
3878 return Register();
3879}
3880
3881Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(MVT RetVT, Register Op0) {
3882 if (RetVT.SimpleTy != MVT::v8bf16)
3883 return Register();
3884 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3885 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
3886 }
3887 return Register();
3888}
3889
3890Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(MVT RetVT, Register Op0) {
3891 if (RetVT.SimpleTy != MVT::v16bf16)
3892 return Register();
3893 if ((Subtarget->hasBF16())) {
3894 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
3895 }
3896 return Register();
3897}
3898
3899Register fastEmit_X86ISD_CVTNEPS2BF16_r(MVT VT, MVT RetVT, Register Op0) {
3900 switch (VT.SimpleTy) {
3901 case MVT::v4f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(RetVT, Op0);
3902 case MVT::v8f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(RetVT, Op0);
3903 case MVT::v16f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(RetVT, Op0);
3904 default: return Register();
3905 }
3906}
3907
3908// FastEmit functions for X86ISD::CVTP2IBS.
3909
3910Register fastEmit_X86ISD_CVTP2IBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
3911 if (RetVT.SimpleTy != MVT::v8i16)
3912 return Register();
3913 if ((Subtarget->hasAVX10_2())) {
3914 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
3915 }
3916 return Register();
3917}
3918
3919Register fastEmit_X86ISD_CVTP2IBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
3920 if (RetVT.SimpleTy != MVT::v16i16)
3921 return Register();
3922 if ((Subtarget->hasAVX10_2())) {
3923 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
3924 }
3925 return Register();
3926}
3927
3928Register fastEmit_X86ISD_CVTP2IBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
3929 if (RetVT.SimpleTy != MVT::v32i16)
3930 return Register();
3931 if ((Subtarget->hasAVX10_2())) {
3932 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZrr, RC: &X86::VR512RegClass, Op0);
3933 }
3934 return Register();
3935}
3936
3937Register fastEmit_X86ISD_CVTP2IBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
3938 if (RetVT.SimpleTy != MVT::v8i16)
3939 return Register();
3940 if ((Subtarget->hasAVX10_2())) {
3941 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
3942 }
3943 return Register();
3944}
3945
3946Register fastEmit_X86ISD_CVTP2IBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
3947 if (RetVT.SimpleTy != MVT::v16i16)
3948 return Register();
3949 if ((Subtarget->hasAVX10_2())) {
3950 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
3951 }
3952 return Register();
3953}
3954
3955Register fastEmit_X86ISD_CVTP2IBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
3956 if (RetVT.SimpleTy != MVT::v32i16)
3957 return Register();
3958 if ((Subtarget->hasAVX10_2())) {
3959 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZrr, RC: &X86::VR512RegClass, Op0);
3960 }
3961 return Register();
3962}
3963
3964Register fastEmit_X86ISD_CVTP2IBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
3965 if (RetVT.SimpleTy != MVT::v4i32)
3966 return Register();
3967 if ((Subtarget->hasAVX10_2())) {
3968 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
3969 }
3970 return Register();
3971}
3972
3973Register fastEmit_X86ISD_CVTP2IBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
3974 if (RetVT.SimpleTy != MVT::v8i32)
3975 return Register();
3976 if ((Subtarget->hasAVX10_2())) {
3977 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
3978 }
3979 return Register();
3980}
3981
3982Register fastEmit_X86ISD_CVTP2IBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
3983 if (RetVT.SimpleTy != MVT::v16i32)
3984 return Register();
3985 if ((Subtarget->hasAVX10_2())) {
3986 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZrr, RC: &X86::VR512RegClass, Op0);
3987 }
3988 return Register();
3989}
3990
3991Register fastEmit_X86ISD_CVTP2IBS_r(MVT VT, MVT RetVT, Register Op0) {
3992 switch (VT.SimpleTy) {
3993 case MVT::v8f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v8f16_r(RetVT, Op0);
3994 case MVT::v16f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v16f16_r(RetVT, Op0);
3995 case MVT::v32f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v32f16_r(RetVT, Op0);
3996 case MVT::v8bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v8bf16_r(RetVT, Op0);
3997 case MVT::v16bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v16bf16_r(RetVT, Op0);
3998 case MVT::v32bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v32bf16_r(RetVT, Op0);
3999 case MVT::v4f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v4f32_r(RetVT, Op0);
4000 case MVT::v8f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v8f32_r(RetVT, Op0);
4001 case MVT::v16f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v16f32_r(RetVT, Op0);
4002 default: return Register();
4003 }
4004}
4005
4006// FastEmit functions for X86ISD::CVTP2IUBS.
4007
4008Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
4009 if (RetVT.SimpleTy != MVT::v8i16)
4010 return Register();
4011 if ((Subtarget->hasAVX10_2())) {
4012 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4013 }
4014 return Register();
4015}
4016
4017Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4018 if (RetVT.SimpleTy != MVT::v16i16)
4019 return Register();
4020 if ((Subtarget->hasAVX10_2())) {
4021 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4022 }
4023 return Register();
4024}
4025
4026Register fastEmit_X86ISD_CVTP2IUBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4027 if (RetVT.SimpleTy != MVT::v32i16)
4028 return Register();
4029 if ((Subtarget->hasAVX10_2())) {
4030 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZrr, RC: &X86::VR512RegClass, Op0);
4031 }
4032 return Register();
4033}
4034
4035Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
4036 if (RetVT.SimpleTy != MVT::v8i16)
4037 return Register();
4038 if ((Subtarget->hasAVX10_2())) {
4039 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4040 }
4041 return Register();
4042}
4043
4044Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
4045 if (RetVT.SimpleTy != MVT::v16i16)
4046 return Register();
4047 if ((Subtarget->hasAVX10_2())) {
4048 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4049 }
4050 return Register();
4051}
4052
4053Register fastEmit_X86ISD_CVTP2IUBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4054 if (RetVT.SimpleTy != MVT::v32i16)
4055 return Register();
4056 if ((Subtarget->hasAVX10_2())) {
4057 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZrr, RC: &X86::VR512RegClass, Op0);
4058 }
4059 return Register();
4060}
4061
4062Register fastEmit_X86ISD_CVTP2IUBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4063 if (RetVT.SimpleTy != MVT::v4i32)
4064 return Register();
4065 if ((Subtarget->hasAVX10_2())) {
4066 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4067 }
4068 return Register();
4069}
4070
4071Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
4072 if (RetVT.SimpleTy != MVT::v8i32)
4073 return Register();
4074 if ((Subtarget->hasAVX10_2())) {
4075 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4076 }
4077 return Register();
4078}
4079
4080Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
4081 if (RetVT.SimpleTy != MVT::v16i32)
4082 return Register();
4083 if ((Subtarget->hasAVX10_2())) {
4084 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZrr, RC: &X86::VR512RegClass, Op0);
4085 }
4086 return Register();
4087}
4088
4089Register fastEmit_X86ISD_CVTP2IUBS_r(MVT VT, MVT RetVT, Register Op0) {
4090 switch (VT.SimpleTy) {
4091 case MVT::v8f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8f16_r(RetVT, Op0);
4092 case MVT::v16f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16f16_r(RetVT, Op0);
4093 case MVT::v32f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v32f16_r(RetVT, Op0);
4094 case MVT::v8bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8bf16_r(RetVT, Op0);
4095 case MVT::v16bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16bf16_r(RetVT, Op0);
4096 case MVT::v32bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v32bf16_r(RetVT, Op0);
4097 case MVT::v4f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v4f32_r(RetVT, Op0);
4098 case MVT::v8f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8f32_r(RetVT, Op0);
4099 case MVT::v16f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16f32_r(RetVT, Op0);
4100 default: return Register();
4101 }
4102}
4103
4104// FastEmit functions for X86ISD::CVTP2SI.
4105
4106Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
4107 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4108 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
4109 }
4110 return Register();
4111}
4112
4113Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
4114 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4115 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4116 }
4117 return Register();
4118}
4119
4120Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
4121 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4122 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4123 }
4124 return Register();
4125}
4126
4127Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
4128 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4129 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4130 }
4131 return Register();
4132}
4133
4134Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
4135 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4136 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4137 }
4138 return Register();
4139}
4140
4141Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
4142 if ((Subtarget->hasFP16())) {
4143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
4144 }
4145 return Register();
4146}
4147
4148Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4149switch (RetVT.SimpleTy) {
4150 case MVT::v8i16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
4151 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
4152 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
4153 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
4154 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
4155 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
4156 default: return Register();
4157}
4158}
4159
4160Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
4161 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4162 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
4163 }
4164 return Register();
4165}
4166
4167Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
4168 if ((Subtarget->hasFP16())) {
4169 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
4170 }
4171 return Register();
4172}
4173
4174Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
4175switch (RetVT.SimpleTy) {
4176 case MVT::v16i16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
4177 case MVT::v16i32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
4178 default: return Register();
4179}
4180}
4181
4182Register fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
4183 if (RetVT.SimpleTy != MVT::v32i16)
4184 return Register();
4185 if ((Subtarget->hasFP16())) {
4186 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
4187 }
4188 return Register();
4189}
4190
4191Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
4192 if ((Subtarget->hasVLX())) {
4193 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4194 }
4195 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4196 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4197 }
4198 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4199 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4200 }
4201 return Register();
4202}
4203
4204Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
4205 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4206 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4207 }
4208 return Register();
4209}
4210
4211Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
4212 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4213 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4214 }
4215 return Register();
4216}
4217
4218Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4219switch (RetVT.SimpleTy) {
4220 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
4221 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
4222 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
4223 default: return Register();
4224}
4225}
4226
4227Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
4228 if ((Subtarget->hasVLX())) {
4229 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4230 }
4231 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4232 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
4233 }
4234 return Register();
4235}
4236
4237Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
4238 if ((Subtarget->hasDQI())) {
4239 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
4240 }
4241 return Register();
4242}
4243
4244Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
4245switch (RetVT.SimpleTy) {
4246 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
4247 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
4248 default: return Register();
4249}
4250}
4251
4252Register fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
4253 if (RetVT.SimpleTy != MVT::v16i32)
4254 return Register();
4255 if ((Subtarget->hasAVX512())) {
4256 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
4257 }
4258 return Register();
4259}
4260
4261Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
4262 if ((Subtarget->hasVLX())) {
4263 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4264 }
4265 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4266 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4267 }
4268 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4269 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4270 }
4271 return Register();
4272}
4273
4274Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
4275 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4276 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4277 }
4278 return Register();
4279}
4280
4281Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4282switch (RetVT.SimpleTy) {
4283 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
4284 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
4285 default: return Register();
4286}
4287}
4288
4289Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
4290 if ((Subtarget->hasVLX())) {
4291 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
4292 }
4293 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4294 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
4295 }
4296 return Register();
4297}
4298
4299Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
4300 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4301 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4302 }
4303 return Register();
4304}
4305
4306Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
4307switch (RetVT.SimpleTy) {
4308 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
4309 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
4310 default: return Register();
4311}
4312}
4313
4314Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
4315 if ((Subtarget->hasAVX512())) {
4316 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
4317 }
4318 return Register();
4319}
4320
4321Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
4322 if ((Subtarget->hasDQI())) {
4323 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
4324 }
4325 return Register();
4326}
4327
4328Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
4329switch (RetVT.SimpleTy) {
4330 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
4331 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
4332 default: return Register();
4333}
4334}
4335
4336Register fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
4337 switch (VT.SimpleTy) {
4338 case MVT::v8f16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(RetVT, Op0);
4339 case MVT::v16f16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(RetVT, Op0);
4340 case MVT::v32f16: return fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(RetVT, Op0);
4341 case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0);
4342 case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0);
4343 case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0);
4344 case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0);
4345 case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0);
4346 case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0);
4347 default: return Register();
4348 }
4349}
4350
4351// FastEmit functions for X86ISD::CVTP2UI.
4352
4353Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
4354 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4355 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
4356 }
4357 return Register();
4358}
4359
4360Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
4361 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4362 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4363 }
4364 return Register();
4365}
4366
4367Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
4368 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4369 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4370 }
4371 return Register();
4372}
4373
4374Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
4375 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4376 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4377 }
4378 return Register();
4379}
4380
4381Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
4382 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4383 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4384 }
4385 return Register();
4386}
4387
4388Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
4389 if ((Subtarget->hasFP16())) {
4390 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
4391 }
4392 return Register();
4393}
4394
4395Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4396switch (RetVT.SimpleTy) {
4397 case MVT::v8i16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
4398 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
4399 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
4400 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
4401 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
4402 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
4403 default: return Register();
4404}
4405}
4406
4407Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
4408 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4409 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
4410 }
4411 return Register();
4412}
4413
4414Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
4415 if ((Subtarget->hasFP16())) {
4416 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
4417 }
4418 return Register();
4419}
4420
4421Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
4422switch (RetVT.SimpleTy) {
4423 case MVT::v16i16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
4424 case MVT::v16i32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
4425 default: return Register();
4426}
4427}
4428
4429Register fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
4430 if (RetVT.SimpleTy != MVT::v32i16)
4431 return Register();
4432 if ((Subtarget->hasFP16())) {
4433 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
4434 }
4435 return Register();
4436}
4437
4438Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
4439 if ((Subtarget->hasVLX())) {
4440 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4441 }
4442 return Register();
4443}
4444
4445Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
4446 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4447 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4448 }
4449 return Register();
4450}
4451
4452Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
4453 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4455 }
4456 return Register();
4457}
4458
4459Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4460switch (RetVT.SimpleTy) {
4461 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
4462 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
4463 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
4464 default: return Register();
4465}
4466}
4467
4468Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
4469 if ((Subtarget->hasVLX())) {
4470 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4471 }
4472 return Register();
4473}
4474
4475Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
4476 if ((Subtarget->hasDQI())) {
4477 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
4478 }
4479 return Register();
4480}
4481
4482Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
4483switch (RetVT.SimpleTy) {
4484 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
4485 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
4486 default: return Register();
4487}
4488}
4489
4490Register fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
4491 if (RetVT.SimpleTy != MVT::v16i32)
4492 return Register();
4493 if ((Subtarget->hasAVX512())) {
4494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
4495 }
4496 return Register();
4497}
4498
4499Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
4500 if ((Subtarget->hasVLX())) {
4501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4502 }
4503 return Register();
4504}
4505
4506Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
4507 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4508 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4509 }
4510 return Register();
4511}
4512
4513Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4514switch (RetVT.SimpleTy) {
4515 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
4516 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
4517 default: return Register();
4518}
4519}
4520
4521Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
4522 if ((Subtarget->hasVLX())) {
4523 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
4524 }
4525 return Register();
4526}
4527
4528Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
4529 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4530 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4531 }
4532 return Register();
4533}
4534
4535Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
4536switch (RetVT.SimpleTy) {
4537 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
4538 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
4539 default: return Register();
4540}
4541}
4542
4543Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
4544 if ((Subtarget->hasAVX512())) {
4545 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
4546 }
4547 return Register();
4548}
4549
4550Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
4551 if ((Subtarget->hasDQI())) {
4552 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
4553 }
4554 return Register();
4555}
4556
4557Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
4558switch (RetVT.SimpleTy) {
4559 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
4560 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
4561 default: return Register();
4562}
4563}
4564
4565Register fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
4566 switch (VT.SimpleTy) {
4567 case MVT::v8f16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(RetVT, Op0);
4568 case MVT::v16f16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(RetVT, Op0);
4569 case MVT::v32f16: return fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(RetVT, Op0);
4570 case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0);
4571 case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0);
4572 case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0);
4573 case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0);
4574 case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0);
4575 case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0);
4576 default: return Register();
4577 }
4578}
4579
4580// FastEmit functions for X86ISD::CVTPH2PS.
4581
4582Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Register Op0) {
4583 if ((Subtarget->hasVLX())) {
4584 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4585 }
4586 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4587 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
4588 }
4589 return Register();
4590}
4591
4592Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Register Op0) {
4593 if ((Subtarget->hasVLX())) {
4594 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
4595 }
4596 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4597 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
4598 }
4599 return Register();
4600}
4601
4602Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, Register Op0) {
4603switch (RetVT.SimpleTy) {
4604 case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
4605 case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
4606 default: return Register();
4607}
4608}
4609
4610Register fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, Register Op0) {
4611 if (RetVT.SimpleTy != MVT::v16f32)
4612 return Register();
4613 if ((Subtarget->hasAVX512())) {
4614 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
4615 }
4616 return Register();
4617}
4618
4619Register fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, Register Op0) {
4620 switch (VT.SimpleTy) {
4621 case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
4622 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
4623 default: return Register();
4624 }
4625}
4626
4627// FastEmit functions for X86ISD::CVTPH2PS_SAE.
4628
4629Register fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(MVT RetVT, Register Op0) {
4630 if (RetVT.SimpleTy != MVT::v16f32)
4631 return Register();
4632 if ((Subtarget->hasAVX512())) {
4633 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrrb, RC: &X86::VR512RegClass, Op0);
4634 }
4635 return Register();
4636}
4637
4638Register fastEmit_X86ISD_CVTPH2PS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
4639 switch (VT.SimpleTy) {
4640 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(RetVT, Op0);
4641 default: return Register();
4642 }
4643}
4644
4645// FastEmit functions for X86ISD::CVTS2SI.
4646
4647Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Register Op0) {
4648 if ((Subtarget->hasFP16())) {
4649 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4650 }
4651 return Register();
4652}
4653
4654Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Register Op0) {
4655 if ((Subtarget->hasFP16())) {
4656 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4657 }
4658 return Register();
4659}
4660
4661Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4662switch (RetVT.SimpleTy) {
4663 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Op0);
4664 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Op0);
4665 default: return Register();
4666}
4667}
4668
4669Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Register Op0) {
4670 if ((Subtarget->hasAVX512())) {
4671 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4672 }
4673 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4674 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4675 }
4676 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4677 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4678 }
4679 return Register();
4680}
4681
4682Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Register Op0) {
4683 if ((Subtarget->hasAVX512())) {
4684 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4685 }
4686 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4687 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4688 }
4689 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4690 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4691 }
4692 return Register();
4693}
4694
4695Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4696switch (RetVT.SimpleTy) {
4697 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0);
4698 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0);
4699 default: return Register();
4700}
4701}
4702
4703Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Register Op0) {
4704 if ((Subtarget->hasAVX512())) {
4705 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4706 }
4707 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4708 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4709 }
4710 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4711 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4712 }
4713 return Register();
4714}
4715
4716Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Register Op0) {
4717 if ((Subtarget->hasAVX512())) {
4718 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4719 }
4720 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4721 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4722 }
4723 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4724 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4725 }
4726 return Register();
4727}
4728
4729Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4730switch (RetVT.SimpleTy) {
4731 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0);
4732 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0);
4733 default: return Register();
4734}
4735}
4736
4737Register fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, Register Op0) {
4738 switch (VT.SimpleTy) {
4739 case MVT::v8f16: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(RetVT, Op0);
4740 case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0);
4741 case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0);
4742 default: return Register();
4743 }
4744}
4745
4746// FastEmit functions for X86ISD::CVTS2UI.
4747
4748Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Register Op0) {
4749 if ((Subtarget->hasFP16())) {
4750 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4751 }
4752 return Register();
4753}
4754
4755Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Register Op0) {
4756 if ((Subtarget->hasFP16())) {
4757 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4758 }
4759 return Register();
4760}
4761
4762Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4763switch (RetVT.SimpleTy) {
4764 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Op0);
4765 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Op0);
4766 default: return Register();
4767}
4768}
4769
4770Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Register Op0) {
4771 if ((Subtarget->hasAVX512())) {
4772 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4773 }
4774 return Register();
4775}
4776
4777Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Register Op0) {
4778 if ((Subtarget->hasAVX512())) {
4779 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4780 }
4781 return Register();
4782}
4783
4784Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4785switch (RetVT.SimpleTy) {
4786 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0);
4787 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0);
4788 default: return Register();
4789}
4790}
4791
4792Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Register Op0) {
4793 if ((Subtarget->hasAVX512())) {
4794 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4795 }
4796 return Register();
4797}
4798
4799Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Register Op0) {
4800 if ((Subtarget->hasAVX512())) {
4801 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4802 }
4803 return Register();
4804}
4805
4806Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4807switch (RetVT.SimpleTy) {
4808 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0);
4809 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0);
4810 default: return Register();
4811}
4812}
4813
4814Register fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, Register Op0) {
4815 switch (VT.SimpleTy) {
4816 case MVT::v8f16: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(RetVT, Op0);
4817 case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0);
4818 case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0);
4819 default: return Register();
4820 }
4821}
4822
4823// FastEmit functions for X86ISD::CVTSI2P.
4824
4825Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
4826 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4827 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4828 }
4829 return Register();
4830}
4831
4832Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
4833 if ((Subtarget->hasVLX())) {
4834 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
4835 }
4836 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4837 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4838 }
4839 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4840 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4841 }
4842 return Register();
4843}
4844
4845Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
4846switch (RetVT.SimpleTy) {
4847 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
4848 case MVT::v2f64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
4849 default: return Register();
4850}
4851}
4852
4853Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
4854 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4855 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4856 }
4857 return Register();
4858}
4859
4860Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
4861 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4862 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4863 }
4864 return Register();
4865}
4866
4867Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
4868switch (RetVT.SimpleTy) {
4869 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
4870 case MVT::v4f32: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
4871 default: return Register();
4872}
4873}
4874
4875Register fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
4876 if (RetVT.SimpleTy != MVT::v8f16)
4877 return Register();
4878 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4879 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
4880 }
4881 return Register();
4882}
4883
4884Register fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, Register Op0) {
4885 switch (VT.SimpleTy) {
4886 case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
4887 case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
4888 case MVT::v4i64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
4889 default: return Register();
4890 }
4891}
4892
4893// FastEmit functions for X86ISD::CVTTP2IBS.
4894
4895Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
4896 if (RetVT.SimpleTy != MVT::v8i16)
4897 return Register();
4898 if ((Subtarget->hasAVX10_2())) {
4899 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4900 }
4901 return Register();
4902}
4903
4904Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4905 if (RetVT.SimpleTy != MVT::v16i16)
4906 return Register();
4907 if ((Subtarget->hasAVX10_2())) {
4908 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4909 }
4910 return Register();
4911}
4912
4913Register fastEmit_X86ISD_CVTTP2IBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4914 if (RetVT.SimpleTy != MVT::v32i16)
4915 return Register();
4916 if ((Subtarget->hasAVX10_2())) {
4917 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZrr, RC: &X86::VR512RegClass, Op0);
4918 }
4919 return Register();
4920}
4921
4922Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
4923 if (RetVT.SimpleTy != MVT::v8i16)
4924 return Register();
4925 if ((Subtarget->hasAVX10_2())) {
4926 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4927 }
4928 return Register();
4929}
4930
4931Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
4932 if (RetVT.SimpleTy != MVT::v16i16)
4933 return Register();
4934 if ((Subtarget->hasAVX10_2())) {
4935 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4936 }
4937 return Register();
4938}
4939
4940Register fastEmit_X86ISD_CVTTP2IBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4941 if (RetVT.SimpleTy != MVT::v32i16)
4942 return Register();
4943 if ((Subtarget->hasAVX10_2())) {
4944 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZrr, RC: &X86::VR512RegClass, Op0);
4945 }
4946 return Register();
4947}
4948
4949Register fastEmit_X86ISD_CVTTP2IBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4950 if (RetVT.SimpleTy != MVT::v4i32)
4951 return Register();
4952 if ((Subtarget->hasAVX10_2())) {
4953 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4954 }
4955 return Register();
4956}
4957
4958Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
4959 if (RetVT.SimpleTy != MVT::v8i32)
4960 return Register();
4961 if ((Subtarget->hasAVX10_2())) {
4962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4963 }
4964 return Register();
4965}
4966
4967Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
4968 if (RetVT.SimpleTy != MVT::v16i32)
4969 return Register();
4970 if ((Subtarget->hasAVX10_2())) {
4971 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZrr, RC: &X86::VR512RegClass, Op0);
4972 }
4973 return Register();
4974}
4975
4976Register fastEmit_X86ISD_CVTTP2IBS_r(MVT VT, MVT RetVT, Register Op0) {
4977 switch (VT.SimpleTy) {
4978 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8f16_r(RetVT, Op0);
4979 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16f16_r(RetVT, Op0);
4980 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v32f16_r(RetVT, Op0);
4981 case MVT::v8bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8bf16_r(RetVT, Op0);
4982 case MVT::v16bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16bf16_r(RetVT, Op0);
4983 case MVT::v32bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v32bf16_r(RetVT, Op0);
4984 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v4f32_r(RetVT, Op0);
4985 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8f32_r(RetVT, Op0);
4986 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16f32_r(RetVT, Op0);
4987 default: return Register();
4988 }
4989}
4990
4991// FastEmit functions for X86ISD::CVTTP2IBS_SAE.
4992
4993Register fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
4994 if (RetVT.SimpleTy != MVT::v32i16)
4995 return Register();
4996 if ((Subtarget->hasAVX10_2())) {
4997 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZrrb, RC: &X86::VR512RegClass, Op0);
4998 }
4999 return Register();
5000}
5001
5002Register fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5003 if (RetVT.SimpleTy != MVT::v16i32)
5004 return Register();
5005 if ((Subtarget->hasAVX10_2())) {
5006 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZrrb, RC: &X86::VR512RegClass, Op0);
5007 }
5008 return Register();
5009}
5010
5011Register fastEmit_X86ISD_CVTTP2IBS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5012 switch (VT.SimpleTy) {
5013 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v32f16_r(RetVT, Op0);
5014 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v16f32_r(RetVT, Op0);
5015 default: return Register();
5016 }
5017}
5018
5019// FastEmit functions for X86ISD::CVTTP2IUBS.
5020
5021Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
5022 if (RetVT.SimpleTy != MVT::v8i16)
5023 return Register();
5024 if ((Subtarget->hasAVX10_2())) {
5025 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5026 }
5027 return Register();
5028}
5029
5030Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
5031 if (RetVT.SimpleTy != MVT::v16i16)
5032 return Register();
5033 if ((Subtarget->hasAVX10_2())) {
5034 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5035 }
5036 return Register();
5037}
5038
5039Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
5040 if (RetVT.SimpleTy != MVT::v32i16)
5041 return Register();
5042 if ((Subtarget->hasAVX10_2())) {
5043 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZrr, RC: &X86::VR512RegClass, Op0);
5044 }
5045 return Register();
5046}
5047
5048Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
5049 if (RetVT.SimpleTy != MVT::v8i16)
5050 return Register();
5051 if ((Subtarget->hasAVX10_2())) {
5052 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5053 }
5054 return Register();
5055}
5056
5057Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
5058 if (RetVT.SimpleTy != MVT::v16i16)
5059 return Register();
5060 if ((Subtarget->hasAVX10_2())) {
5061 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5062 }
5063 return Register();
5064}
5065
5066Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
5067 if (RetVT.SimpleTy != MVT::v32i16)
5068 return Register();
5069 if ((Subtarget->hasAVX10_2())) {
5070 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZrr, RC: &X86::VR512RegClass, Op0);
5071 }
5072 return Register();
5073}
5074
5075Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5076 if (RetVT.SimpleTy != MVT::v4i32)
5077 return Register();
5078 if ((Subtarget->hasAVX10_2())) {
5079 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5080 }
5081 return Register();
5082}
5083
5084Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5085 if (RetVT.SimpleTy != MVT::v8i32)
5086 return Register();
5087 if ((Subtarget->hasAVX10_2())) {
5088 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5089 }
5090 return Register();
5091}
5092
5093Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5094 if (RetVT.SimpleTy != MVT::v16i32)
5095 return Register();
5096 if ((Subtarget->hasAVX10_2())) {
5097 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZrr, RC: &X86::VR512RegClass, Op0);
5098 }
5099 return Register();
5100}
5101
5102Register fastEmit_X86ISD_CVTTP2IUBS_r(MVT VT, MVT RetVT, Register Op0) {
5103 switch (VT.SimpleTy) {
5104 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f16_r(RetVT, Op0);
5105 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f16_r(RetVT, Op0);
5106 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v32f16_r(RetVT, Op0);
5107 case MVT::v8bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8bf16_r(RetVT, Op0);
5108 case MVT::v16bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16bf16_r(RetVT, Op0);
5109 case MVT::v32bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v32bf16_r(RetVT, Op0);
5110 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v4f32_r(RetVT, Op0);
5111 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f32_r(RetVT, Op0);
5112 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f32_r(RetVT, Op0);
5113 default: return Register();
5114 }
5115}
5116
5117// FastEmit functions for X86ISD::CVTTP2IUBS_SAE.
5118
5119Register fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5120 if (RetVT.SimpleTy != MVT::v32i16)
5121 return Register();
5122 if ((Subtarget->hasAVX10_2())) {
5123 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZrrb, RC: &X86::VR512RegClass, Op0);
5124 }
5125 return Register();
5126}
5127
5128Register fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5129 if (RetVT.SimpleTy != MVT::v16i32)
5130 return Register();
5131 if ((Subtarget->hasAVX10_2())) {
5132 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZrrb, RC: &X86::VR512RegClass, Op0);
5133 }
5134 return Register();
5135}
5136
5137Register fastEmit_X86ISD_CVTTP2IUBS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5138 switch (VT.SimpleTy) {
5139 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v32f16_r(RetVT, Op0);
5140 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v16f32_r(RetVT, Op0);
5141 default: return Register();
5142 }
5143}
5144
5145// FastEmit functions for X86ISD::CVTTP2SI.
5146
5147Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
5148 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5149 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
5150 }
5151 return Register();
5152}
5153
5154Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
5155 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5156 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5157 }
5158 return Register();
5159}
5160
5161Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
5162 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5163 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
5164 }
5165 return Register();
5166}
5167
5168Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
5169 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5170 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5171 }
5172 return Register();
5173}
5174
5175Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
5176 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5177 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5178 }
5179 return Register();
5180}
5181
5182Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
5183 if ((Subtarget->hasFP16())) {
5184 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
5185 }
5186 return Register();
5187}
5188
5189Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
5190switch (RetVT.SimpleTy) {
5191 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
5192 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
5193 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
5194 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
5195 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
5196 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
5197 default: return Register();
5198}
5199}
5200
5201Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
5202 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5203 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
5204 }
5205 return Register();
5206}
5207
5208Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
5209 if ((Subtarget->hasFP16())) {
5210 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
5211 }
5212 return Register();
5213}
5214
5215Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
5216switch (RetVT.SimpleTy) {
5217 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
5218 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
5219 default: return Register();
5220}
5221}
5222
5223Register fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
5224 if (RetVT.SimpleTy != MVT::v32i16)
5225 return Register();
5226 if ((Subtarget->hasFP16())) {
5227 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
5228 }
5229 return Register();
5230}
5231
5232Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5233 if ((Subtarget->hasVLX())) {
5234 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5235 }
5236 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5237 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
5238 }
5239 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5240 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
5241 }
5242 return Register();
5243}
5244
5245Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5246 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5247 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5248 }
5249 return Register();
5250}
5251
5252Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5253 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5254 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5255 }
5256 return Register();
5257}
5258
5259Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
5260switch (RetVT.SimpleTy) {
5261 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
5262 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
5263 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
5264 default: return Register();
5265}
5266}
5267
5268Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5269 if ((Subtarget->hasVLX())) {
5270 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
5271 }
5272 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5273 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
5274 }
5275 return Register();
5276}
5277
5278Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5279 if ((Subtarget->hasDQI())) {
5280 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
5281 }
5282 return Register();
5283}
5284
5285Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
5286switch (RetVT.SimpleTy) {
5287 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
5288 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
5289 default: return Register();
5290}
5291}
5292
5293Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
5294 if (RetVT.SimpleTy != MVT::v16i32)
5295 return Register();
5296 if ((Subtarget->hasAVX512())) {
5297 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
5298 }
5299 return Register();
5300}
5301
5302Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5303 if ((Subtarget->hasVLX())) {
5304 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5305 }
5306 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5307 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
5308 }
5309 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5310 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
5311 }
5312 return Register();
5313}
5314
5315Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5316 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5317 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5318 }
5319 return Register();
5320}
5321
5322Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
5323switch (RetVT.SimpleTy) {
5324 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
5325 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
5326 default: return Register();
5327}
5328}
5329
5330Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5331 if ((Subtarget->hasVLX())) {
5332 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
5333 }
5334 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5335 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
5336 }
5337 return Register();
5338}
5339
5340Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5341 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5342 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5343 }
5344 return Register();
5345}
5346
5347Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
5348switch (RetVT.SimpleTy) {
5349 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
5350 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
5351 default: return Register();
5352}
5353}
5354
5355Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5356 if ((Subtarget->hasAVX512())) {
5357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
5358 }
5359 return Register();
5360}
5361
5362Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5363 if ((Subtarget->hasDQI())) {
5364 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
5365 }
5366 return Register();
5367}
5368
5369Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
5370switch (RetVT.SimpleTy) {
5371 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
5372 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
5373 default: return Register();
5374}
5375}
5376
5377Register fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
5378 switch (VT.SimpleTy) {
5379 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
5380 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
5381 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
5382 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
5383 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
5384 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
5385 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
5386 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
5387 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
5388 default: return Register();
5389 }
5390}
5391
5392// FastEmit functions for X86ISD::CVTTP2SIS.
5393
5394Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5395 if ((Subtarget->hasAVX10_2())) {
5396 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5397 }
5398 return Register();
5399}
5400
5401Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5402 if ((Subtarget->hasAVX10_2())) {
5403 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5404 }
5405 return Register();
5406}
5407
5408Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5409 if ((Subtarget->hasAVX10_2())) {
5410 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5411 }
5412 return Register();
5413}
5414
5415Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5416switch (RetVT.SimpleTy) {
5417 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i32_r(Op0);
5418 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v2i64_r(Op0);
5419 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i64_r(Op0);
5420 default: return Register();
5421}
5422}
5423
5424Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5425 if ((Subtarget->hasAVX10_2())) {
5426 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5427 }
5428 return Register();
5429}
5430
5431Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5432 if ((Subtarget->hasAVX10_2())) {
5433 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZrr, RC: &X86::VR512RegClass, Op0);
5434 }
5435 return Register();
5436}
5437
5438Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5439switch (RetVT.SimpleTy) {
5440 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i32_r(Op0);
5441 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i64_r(Op0);
5442 default: return Register();
5443}
5444}
5445
5446Register fastEmit_X86ISD_CVTTP2SIS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5447 if (RetVT.SimpleTy != MVT::v16i32)
5448 return Register();
5449 if ((Subtarget->hasAVX10_2())) {
5450 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZrr, RC: &X86::VR512RegClass, Op0);
5451 }
5452 return Register();
5453}
5454
5455Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5456 if ((Subtarget->hasAVX10_2())) {
5457 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5458 }
5459 return Register();
5460}
5461
5462Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5463 if ((Subtarget->hasAVX10_2())) {
5464 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5465 }
5466 return Register();
5467}
5468
5469Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
5470switch (RetVT.SimpleTy) {
5471 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v4i32_r(Op0);
5472 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v2i64_r(Op0);
5473 default: return Register();
5474}
5475}
5476
5477Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5478 if ((Subtarget->hasAVX10_2())) {
5479 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ256rr, RC: &X86::VR128XRegClass, Op0);
5480 }
5481 return Register();
5482}
5483
5484Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5485 if ((Subtarget->hasAVX10_2())) {
5486 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5487 }
5488 return Register();
5489}
5490
5491Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_r(MVT RetVT, Register Op0) {
5492switch (RetVT.SimpleTy) {
5493 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i32_r(Op0);
5494 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i64_r(Op0);
5495 default: return Register();
5496}
5497}
5498
5499Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5500 if ((Subtarget->hasAVX10_2())) {
5501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZrr, RC: &X86::VR256XRegClass, Op0);
5502 }
5503 return Register();
5504}
5505
5506Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5507 if ((Subtarget->hasAVX10_2())) {
5508 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZrr, RC: &X86::VR512RegClass, Op0);
5509 }
5510 return Register();
5511}
5512
5513Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_r(MVT RetVT, Register Op0) {
5514switch (RetVT.SimpleTy) {
5515 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i32_r(Op0);
5516 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i64_r(Op0);
5517 default: return Register();
5518}
5519}
5520
5521Register fastEmit_X86ISD_CVTTP2SIS_r(MVT VT, MVT RetVT, Register Op0) {
5522 switch (VT.SimpleTy) {
5523 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_r(RetVT, Op0);
5524 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_r(RetVT, Op0);
5525 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v16f32_r(RetVT, Op0);
5526 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_r(RetVT, Op0);
5527 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_r(RetVT, Op0);
5528 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_r(RetVT, Op0);
5529 default: return Register();
5530 }
5531}
5532
5533// FastEmit functions for X86ISD::CVTTP2SIS_SAE.
5534
5535Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
5536 if (RetVT.SimpleTy != MVT::v4i64)
5537 return Register();
5538 if ((Subtarget->hasAVX10_2())) {
5539 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
5540 }
5541 return Register();
5542}
5543
5544Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
5545 if (RetVT.SimpleTy != MVT::v8i64)
5546 return Register();
5547 if ((Subtarget->hasAVX10_2())) {
5548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZrrb, RC: &X86::VR512RegClass, Op0);
5549 }
5550 return Register();
5551}
5552
5553Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5554 if (RetVT.SimpleTy != MVT::v16i32)
5555 return Register();
5556 if ((Subtarget->hasAVX10_2())) {
5557 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZrrb, RC: &X86::VR512RegClass, Op0);
5558 }
5559 return Register();
5560}
5561
5562Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5563 if ((Subtarget->hasAVX10_2())) {
5564 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ256rrb, RC: &X86::VR128XRegClass, Op0);
5565 }
5566 return Register();
5567}
5568
5569Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5570 if ((Subtarget->hasAVX10_2())) {
5571 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
5572 }
5573 return Register();
5574}
5575
5576Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_r(MVT RetVT, Register Op0) {
5577switch (RetVT.SimpleTy) {
5578 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i32_r(Op0);
5579 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i64_r(Op0);
5580 default: return Register();
5581}
5582}
5583
5584Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5585 if ((Subtarget->hasAVX10_2())) {
5586 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZrrb, RC: &X86::VR256XRegClass, Op0);
5587 }
5588 return Register();
5589}
5590
5591Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5592 if ((Subtarget->hasAVX10_2())) {
5593 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZrrb, RC: &X86::VR512RegClass, Op0);
5594 }
5595 return Register();
5596}
5597
5598Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
5599switch (RetVT.SimpleTy) {
5600 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5601 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5602 default: return Register();
5603}
5604}
5605
5606Register fastEmit_X86ISD_CVTTP2SIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5607 switch (VT.SimpleTy) {
5608 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f32_r(RetVT, Op0);
5609 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f32_r(RetVT, Op0);
5610 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v16f32_r(RetVT, Op0);
5611 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_r(RetVT, Op0);
5612 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_r(RetVT, Op0);
5613 default: return Register();
5614 }
5615}
5616
5617// FastEmit functions for X86ISD::CVTTP2SI_SAE.
5618
5619Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
5620 if (RetVT.SimpleTy != MVT::v8i64)
5621 return Register();
5622 if ((Subtarget->hasFP16())) {
5623 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrrb, RC: &X86::VR512RegClass, Op0);
5624 }
5625 return Register();
5626}
5627
5628Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
5629 if (RetVT.SimpleTy != MVT::v16i32)
5630 return Register();
5631 if ((Subtarget->hasFP16())) {
5632 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrrb, RC: &X86::VR512RegClass, Op0);
5633 }
5634 return Register();
5635}
5636
5637Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5638 if (RetVT.SimpleTy != MVT::v32i16)
5639 return Register();
5640 if ((Subtarget->hasFP16())) {
5641 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrrb, RC: &X86::VR512RegClass, Op0);
5642 }
5643 return Register();
5644}
5645
5646Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
5647 if (RetVT.SimpleTy != MVT::v8i64)
5648 return Register();
5649 if ((Subtarget->hasDQI())) {
5650 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrrb, RC: &X86::VR512RegClass, Op0);
5651 }
5652 return Register();
5653}
5654
5655Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5656 if (RetVT.SimpleTy != MVT::v16i32)
5657 return Register();
5658 if ((Subtarget->hasAVX512())) {
5659 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrrb, RC: &X86::VR512RegClass, Op0);
5660 }
5661 return Register();
5662}
5663
5664Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5665 if ((Subtarget->hasAVX512())) {
5666 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrrb, RC: &X86::VR256XRegClass, Op0);
5667 }
5668 return Register();
5669}
5670
5671Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5672 if ((Subtarget->hasDQI())) {
5673 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrrb, RC: &X86::VR512RegClass, Op0);
5674 }
5675 return Register();
5676}
5677
5678Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
5679switch (RetVT.SimpleTy) {
5680 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5681 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5682 default: return Register();
5683}
5684}
5685
5686Register fastEmit_X86ISD_CVTTP2SI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5687 switch (VT.SimpleTy) {
5688 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(RetVT, Op0);
5689 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(RetVT, Op0);
5690 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(RetVT, Op0);
5691 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(RetVT, Op0);
5692 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(RetVT, Op0);
5693 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(RetVT, Op0);
5694 default: return Register();
5695 }
5696}
5697
5698// FastEmit functions for X86ISD::CVTTP2UI.
5699
5700Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
5701 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5702 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
5703 }
5704 return Register();
5705}
5706
5707Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
5708 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5709 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5710 }
5711 return Register();
5712}
5713
5714Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
5715 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5716 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5717 }
5718 return Register();
5719}
5720
5721Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
5722 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5723 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5724 }
5725 return Register();
5726}
5727
5728Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
5729 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5730 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5731 }
5732 return Register();
5733}
5734
5735Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
5736 if ((Subtarget->hasFP16())) {
5737 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
5738 }
5739 return Register();
5740}
5741
5742Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
5743switch (RetVT.SimpleTy) {
5744 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
5745 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
5746 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
5747 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
5748 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
5749 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
5750 default: return Register();
5751}
5752}
5753
5754Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
5755 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5756 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
5757 }
5758 return Register();
5759}
5760
5761Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
5762 if ((Subtarget->hasFP16())) {
5763 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
5764 }
5765 return Register();
5766}
5767
5768Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
5769switch (RetVT.SimpleTy) {
5770 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
5771 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
5772 default: return Register();
5773}
5774}
5775
5776Register fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
5777 if (RetVT.SimpleTy != MVT::v32i16)
5778 return Register();
5779 if ((Subtarget->hasFP16())) {
5780 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
5781 }
5782 return Register();
5783}
5784
5785Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5786 if ((Subtarget->hasVLX())) {
5787 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5788 }
5789 return Register();
5790}
5791
5792Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5793 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5794 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5795 }
5796 return Register();
5797}
5798
5799Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5800 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5801 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5802 }
5803 return Register();
5804}
5805
5806Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
5807switch (RetVT.SimpleTy) {
5808 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
5809 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
5810 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
5811 default: return Register();
5812}
5813}
5814
5815Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5816 if ((Subtarget->hasVLX())) {
5817 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5818 }
5819 return Register();
5820}
5821
5822Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5823 if ((Subtarget->hasDQI())) {
5824 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
5825 }
5826 return Register();
5827}
5828
5829Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
5830switch (RetVT.SimpleTy) {
5831 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
5832 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
5833 default: return Register();
5834}
5835}
5836
5837Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
5838 if (RetVT.SimpleTy != MVT::v16i32)
5839 return Register();
5840 if ((Subtarget->hasAVX512())) {
5841 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
5842 }
5843 return Register();
5844}
5845
5846Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5847 if ((Subtarget->hasVLX())) {
5848 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5849 }
5850 return Register();
5851}
5852
5853Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5854 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5855 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5856 }
5857 return Register();
5858}
5859
5860Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
5861switch (RetVT.SimpleTy) {
5862 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
5863 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
5864 default: return Register();
5865}
5866}
5867
5868Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5869 if ((Subtarget->hasVLX())) {
5870 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
5871 }
5872 return Register();
5873}
5874
5875Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5876 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5877 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5878 }
5879 return Register();
5880}
5881
5882Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
5883switch (RetVT.SimpleTy) {
5884 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
5885 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
5886 default: return Register();
5887}
5888}
5889
5890Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5891 if ((Subtarget->hasAVX512())) {
5892 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
5893 }
5894 return Register();
5895}
5896
5897Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5898 if ((Subtarget->hasDQI())) {
5899 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
5900 }
5901 return Register();
5902}
5903
5904Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
5905switch (RetVT.SimpleTy) {
5906 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
5907 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
5908 default: return Register();
5909}
5910}
5911
5912Register fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
5913 switch (VT.SimpleTy) {
5914 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
5915 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
5916 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
5917 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
5918 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
5919 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
5920 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
5921 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
5922 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
5923 default: return Register();
5924 }
5925}
5926
5927// FastEmit functions for X86ISD::CVTTP2UIS.
5928
5929Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5930 if ((Subtarget->hasAVX10_2())) {
5931 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5932 }
5933 return Register();
5934}
5935
5936Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5937 if ((Subtarget->hasAVX10_2())) {
5938 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5939 }
5940 return Register();
5941}
5942
5943Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5944 if ((Subtarget->hasAVX10_2())) {
5945 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5946 }
5947 return Register();
5948}
5949
5950Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5951switch (RetVT.SimpleTy) {
5952 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i32_r(Op0);
5953 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v2i64_r(Op0);
5954 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i64_r(Op0);
5955 default: return Register();
5956}
5957}
5958
5959Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5960 if ((Subtarget->hasAVX10_2())) {
5961 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5962 }
5963 return Register();
5964}
5965
5966Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5967 if ((Subtarget->hasAVX10_2())) {
5968 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZrr, RC: &X86::VR512RegClass, Op0);
5969 }
5970 return Register();
5971}
5972
5973Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5974switch (RetVT.SimpleTy) {
5975 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i32_r(Op0);
5976 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i64_r(Op0);
5977 default: return Register();
5978}
5979}
5980
5981Register fastEmit_X86ISD_CVTTP2UIS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5982 if (RetVT.SimpleTy != MVT::v16i32)
5983 return Register();
5984 if ((Subtarget->hasAVX10_2())) {
5985 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZrr, RC: &X86::VR512RegClass, Op0);
5986 }
5987 return Register();
5988}
5989
5990Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5991 if ((Subtarget->hasAVX10_2())) {
5992 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5993 }
5994 return Register();
5995}
5996
5997Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5998 if ((Subtarget->hasAVX10_2())) {
5999 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6000 }
6001 return Register();
6002}
6003
6004Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6005switch (RetVT.SimpleTy) {
6006 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v4i32_r(Op0);
6007 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v2i64_r(Op0);
6008 default: return Register();
6009}
6010}
6011
6012Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i32_r(Register Op0) {
6013 if ((Subtarget->hasAVX10_2())) {
6014 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ256rr, RC: &X86::VR128XRegClass, Op0);
6015 }
6016 return Register();
6017}
6018
6019Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i64_r(Register Op0) {
6020 if ((Subtarget->hasAVX10_2())) {
6021 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ256rr, RC: &X86::VR256XRegClass, Op0);
6022 }
6023 return Register();
6024}
6025
6026Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_r(MVT RetVT, Register Op0) {
6027switch (RetVT.SimpleTy) {
6028 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i32_r(Op0);
6029 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i64_r(Op0);
6030 default: return Register();
6031}
6032}
6033
6034Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6035 if ((Subtarget->hasAVX10_2())) {
6036 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZrr, RC: &X86::VR256XRegClass, Op0);
6037 }
6038 return Register();
6039}
6040
6041Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6042 if ((Subtarget->hasAVX10_2())) {
6043 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZrr, RC: &X86::VR512RegClass, Op0);
6044 }
6045 return Register();
6046}
6047
6048Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_r(MVT RetVT, Register Op0) {
6049switch (RetVT.SimpleTy) {
6050 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i32_r(Op0);
6051 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i64_r(Op0);
6052 default: return Register();
6053}
6054}
6055
6056Register fastEmit_X86ISD_CVTTP2UIS_r(MVT VT, MVT RetVT, Register Op0) {
6057 switch (VT.SimpleTy) {
6058 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_r(RetVT, Op0);
6059 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_r(RetVT, Op0);
6060 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v16f32_r(RetVT, Op0);
6061 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_r(RetVT, Op0);
6062 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_r(RetVT, Op0);
6063 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_r(RetVT, Op0);
6064 default: return Register();
6065 }
6066}
6067
6068// FastEmit functions for X86ISD::CVTTP2UIS_SAE.
6069
6070Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6071 if (RetVT.SimpleTy != MVT::v4i64)
6072 return Register();
6073 if ((Subtarget->hasAVX10_2())) {
6074 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
6075 }
6076 return Register();
6077}
6078
6079Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
6080 if (RetVT.SimpleTy != MVT::v8i64)
6081 return Register();
6082 if ((Subtarget->hasAVX10_2())) {
6083 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZrrb, RC: &X86::VR512RegClass, Op0);
6084 }
6085 return Register();
6086}
6087
6088Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6089 if (RetVT.SimpleTy != MVT::v16i32)
6090 return Register();
6091 if ((Subtarget->hasAVX10_2())) {
6092 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZrrb, RC: &X86::VR512RegClass, Op0);
6093 }
6094 return Register();
6095}
6096
6097Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i32_r(Register Op0) {
6098 if ((Subtarget->hasAVX10_2())) {
6099 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ256rrb, RC: &X86::VR128XRegClass, Op0);
6100 }
6101 return Register();
6102}
6103
6104Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i64_r(Register Op0) {
6105 if ((Subtarget->hasAVX10_2())) {
6106 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
6107 }
6108 return Register();
6109}
6110
6111Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_r(MVT RetVT, Register Op0) {
6112switch (RetVT.SimpleTy) {
6113 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i32_r(Op0);
6114 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i64_r(Op0);
6115 default: return Register();
6116}
6117}
6118
6119Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6120 if ((Subtarget->hasAVX10_2())) {
6121 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZrrb, RC: &X86::VR256XRegClass, Op0);
6122 }
6123 return Register();
6124}
6125
6126Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6127 if ((Subtarget->hasAVX10_2())) {
6128 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZrrb, RC: &X86::VR512RegClass, Op0);
6129 }
6130 return Register();
6131}
6132
6133Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6134switch (RetVT.SimpleTy) {
6135 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
6136 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
6137 default: return Register();
6138}
6139}
6140
6141Register fastEmit_X86ISD_CVTTP2UIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6142 switch (VT.SimpleTy) {
6143 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f32_r(RetVT, Op0);
6144 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f32_r(RetVT, Op0);
6145 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v16f32_r(RetVT, Op0);
6146 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_r(RetVT, Op0);
6147 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_r(RetVT, Op0);
6148 default: return Register();
6149 }
6150}
6151
6152// FastEmit functions for X86ISD::CVTTP2UI_SAE.
6153
6154Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6155 if (RetVT.SimpleTy != MVT::v8i64)
6156 return Register();
6157 if ((Subtarget->hasFP16())) {
6158 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6159 }
6160 return Register();
6161}
6162
6163Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
6164 if (RetVT.SimpleTy != MVT::v16i32)
6165 return Register();
6166 if ((Subtarget->hasFP16())) {
6167 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrrb, RC: &X86::VR512RegClass, Op0);
6168 }
6169 return Register();
6170}
6171
6172Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
6173 if (RetVT.SimpleTy != MVT::v32i16)
6174 return Register();
6175 if ((Subtarget->hasFP16())) {
6176 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrrb, RC: &X86::VR512RegClass, Op0);
6177 }
6178 return Register();
6179}
6180
6181Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
6182 if (RetVT.SimpleTy != MVT::v8i64)
6183 return Register();
6184 if ((Subtarget->hasDQI())) {
6185 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6186 }
6187 return Register();
6188}
6189
6190Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6191 if (RetVT.SimpleTy != MVT::v16i32)
6192 return Register();
6193 if ((Subtarget->hasAVX512())) {
6194 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrrb, RC: &X86::VR512RegClass, Op0);
6195 }
6196 return Register();
6197}
6198
6199Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6200 if ((Subtarget->hasAVX512())) {
6201 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrrb, RC: &X86::VR256XRegClass, Op0);
6202 }
6203 return Register();
6204}
6205
6206Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6207 if ((Subtarget->hasDQI())) {
6208 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6209 }
6210 return Register();
6211}
6212
6213Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6214switch (RetVT.SimpleTy) {
6215 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
6216 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
6217 default: return Register();
6218}
6219}
6220
6221Register fastEmit_X86ISD_CVTTP2UI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6222 switch (VT.SimpleTy) {
6223 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(RetVT, Op0);
6224 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(RetVT, Op0);
6225 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(RetVT, Op0);
6226 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(RetVT, Op0);
6227 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(RetVT, Op0);
6228 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(RetVT, Op0);
6229 default: return Register();
6230 }
6231}
6232
6233// FastEmit functions for X86ISD::CVTTS2SI.
6234
6235Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Register Op0) {
6236 if ((Subtarget->hasFP16())) {
6237 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6238 }
6239 return Register();
6240}
6241
6242Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Register Op0) {
6243 if ((Subtarget->hasFP16())) {
6244 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6245 }
6246 return Register();
6247}
6248
6249Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
6250switch (RetVT.SimpleTy) {
6251 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Op0);
6252 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Op0);
6253 default: return Register();
6254}
6255}
6256
6257Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Register Op0) {
6258 if ((Subtarget->hasAVX512())) {
6259 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6260 }
6261 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6262 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6263 }
6264 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6265 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6266 }
6267 return Register();
6268}
6269
6270Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Register Op0) {
6271 if ((Subtarget->hasAVX512())) {
6272 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6273 }
6274 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6275 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6276 }
6277 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6278 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6279 }
6280 return Register();
6281}
6282
6283Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
6284switch (RetVT.SimpleTy) {
6285 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0);
6286 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0);
6287 default: return Register();
6288}
6289}
6290
6291Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Register Op0) {
6292 if ((Subtarget->hasAVX512())) {
6293 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6294 }
6295 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6296 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6297 }
6298 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6299 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6300 }
6301 return Register();
6302}
6303
6304Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Register Op0) {
6305 if ((Subtarget->hasAVX512())) {
6306 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6307 }
6308 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6309 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6310 }
6311 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6312 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6313 }
6314 return Register();
6315}
6316
6317Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
6318switch (RetVT.SimpleTy) {
6319 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0);
6320 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0);
6321 default: return Register();
6322}
6323}
6324
6325Register fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, Register Op0) {
6326 switch (VT.SimpleTy) {
6327 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(RetVT, Op0);
6328 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0);
6329 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0);
6330 default: return Register();
6331 }
6332}
6333
6334// FastEmit functions for X86ISD::CVTTS2SIS.
6335
6336Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i32_r(Register Op0) {
6337 if ((Subtarget->hasAVX10_2())) {
6338 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SISrr_Int, RC: &X86::GR32RegClass, Op0);
6339 }
6340 return Register();
6341}
6342
6343Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i64_r(Register Op0) {
6344 if ((Subtarget->hasAVX10_2())) {
6345 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6346 }
6347 return Register();
6348}
6349
6350Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6351switch (RetVT.SimpleTy) {
6352 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i32_r(Op0);
6353 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i64_r(Op0);
6354 default: return Register();
6355}
6356}
6357
6358Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i32_r(Register Op0) {
6359 if ((Subtarget->hasAVX10_2())) {
6360 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SISrr_Int, RC: &X86::GR32RegClass, Op0);
6361 }
6362 return Register();
6363}
6364
6365Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i64_r(Register Op0) {
6366 if ((Subtarget->hasAVX10_2())) {
6367 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6368 }
6369 return Register();
6370}
6371
6372Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6373switch (RetVT.SimpleTy) {
6374 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i32_r(Op0);
6375 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i64_r(Op0);
6376 default: return Register();
6377}
6378}
6379
6380Register fastEmit_X86ISD_CVTTS2SIS_r(MVT VT, MVT RetVT, Register Op0) {
6381 switch (VT.SimpleTy) {
6382 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_r(RetVT, Op0);
6383 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_r(RetVT, Op0);
6384 default: return Register();
6385 }
6386}
6387
6388// FastEmit functions for X86ISD::CVTTS2SIS_SAE.
6389
6390Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6391 if ((Subtarget->hasAVX10_2())) {
6392 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SISrrb_Int, RC: &X86::GR32RegClass, Op0);
6393 }
6394 return Register();
6395}
6396
6397Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6398 if ((Subtarget->hasAVX10_2())) {
6399 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6400 }
6401 return Register();
6402}
6403
6404Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6405switch (RetVT.SimpleTy) {
6406 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i32_r(Op0);
6407 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i64_r(Op0);
6408 default: return Register();
6409}
6410}
6411
6412Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6413 if ((Subtarget->hasAVX10_2())) {
6414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SISrrb_Int, RC: &X86::GR32RegClass, Op0);
6415 }
6416 return Register();
6417}
6418
6419Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6420 if ((Subtarget->hasAVX10_2())) {
6421 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6422 }
6423 return Register();
6424}
6425
6426Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6427switch (RetVT.SimpleTy) {
6428 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i32_r(Op0);
6429 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i64_r(Op0);
6430 default: return Register();
6431}
6432}
6433
6434Register fastEmit_X86ISD_CVTTS2SIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6435 switch (VT.SimpleTy) {
6436 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_r(RetVT, Op0);
6437 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_r(RetVT, Op0);
6438 default: return Register();
6439 }
6440}
6441
6442// FastEmit functions for X86ISD::CVTTS2SI_SAE.
6443
6444Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Register Op0) {
6445 if ((Subtarget->hasFP16())) {
6446 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6447 }
6448 return Register();
6449}
6450
6451Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Register Op0) {
6452 if ((Subtarget->hasFP16())) {
6453 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6454 }
6455 return Register();
6456}
6457
6458Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6459switch (RetVT.SimpleTy) {
6460 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Op0);
6461 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Op0);
6462 default: return Register();
6463}
6464}
6465
6466Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6467 if ((Subtarget->hasAVX512())) {
6468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6469 }
6470 return Register();
6471}
6472
6473Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6474 if ((Subtarget->hasAVX512())) {
6475 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6476 }
6477 return Register();
6478}
6479
6480Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6481switch (RetVT.SimpleTy) {
6482 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Op0);
6483 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Op0);
6484 default: return Register();
6485}
6486}
6487
6488Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6489 if ((Subtarget->hasAVX512())) {
6490 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6491 }
6492 return Register();
6493}
6494
6495Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6496 if ((Subtarget->hasAVX512())) {
6497 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6498 }
6499 return Register();
6500}
6501
6502Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6503switch (RetVT.SimpleTy) {
6504 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Op0);
6505 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Op0);
6506 default: return Register();
6507}
6508}
6509
6510Register fastEmit_X86ISD_CVTTS2SI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6511 switch (VT.SimpleTy) {
6512 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(RetVT, Op0);
6513 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(RetVT, Op0);
6514 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(RetVT, Op0);
6515 default: return Register();
6516 }
6517}
6518
6519// FastEmit functions for X86ISD::CVTTS2UI.
6520
6521Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Register Op0) {
6522 if ((Subtarget->hasFP16())) {
6523 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6524 }
6525 return Register();
6526}
6527
6528Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Register Op0) {
6529 if ((Subtarget->hasFP16())) {
6530 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6531 }
6532 return Register();
6533}
6534
6535Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
6536switch (RetVT.SimpleTy) {
6537 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Op0);
6538 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Op0);
6539 default: return Register();
6540}
6541}
6542
6543Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Register Op0) {
6544 if ((Subtarget->hasAVX512())) {
6545 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6546 }
6547 return Register();
6548}
6549
6550Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Register Op0) {
6551 if ((Subtarget->hasAVX512())) {
6552 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6553 }
6554 return Register();
6555}
6556
6557Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
6558switch (RetVT.SimpleTy) {
6559 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0);
6560 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0);
6561 default: return Register();
6562}
6563}
6564
6565Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Register Op0) {
6566 if ((Subtarget->hasAVX512())) {
6567 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6568 }
6569 return Register();
6570}
6571
6572Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Register Op0) {
6573 if ((Subtarget->hasAVX512())) {
6574 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6575 }
6576 return Register();
6577}
6578
6579Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
6580switch (RetVT.SimpleTy) {
6581 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0);
6582 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0);
6583 default: return Register();
6584}
6585}
6586
6587Register fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, Register Op0) {
6588 switch (VT.SimpleTy) {
6589 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(RetVT, Op0);
6590 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0);
6591 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0);
6592 default: return Register();
6593 }
6594}
6595
6596// FastEmit functions for X86ISD::CVTTS2UIS.
6597
6598Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i32_r(Register Op0) {
6599 if ((Subtarget->hasAVX10_2())) {
6600 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USISrr_Int, RC: &X86::GR32RegClass, Op0);
6601 }
6602 return Register();
6603}
6604
6605Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i64_r(Register Op0) {
6606 if ((Subtarget->hasAVX10_2())) {
6607 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6608 }
6609 return Register();
6610}
6611
6612Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6613switch (RetVT.SimpleTy) {
6614 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i32_r(Op0);
6615 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i64_r(Op0);
6616 default: return Register();
6617}
6618}
6619
6620Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i32_r(Register Op0) {
6621 if ((Subtarget->hasAVX10_2())) {
6622 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USISrr_Int, RC: &X86::GR32RegClass, Op0);
6623 }
6624 return Register();
6625}
6626
6627Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i64_r(Register Op0) {
6628 if ((Subtarget->hasAVX10_2())) {
6629 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6630 }
6631 return Register();
6632}
6633
6634Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6635switch (RetVT.SimpleTy) {
6636 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i32_r(Op0);
6637 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i64_r(Op0);
6638 default: return Register();
6639}
6640}
6641
6642Register fastEmit_X86ISD_CVTTS2UIS_r(MVT VT, MVT RetVT, Register Op0) {
6643 switch (VT.SimpleTy) {
6644 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_r(RetVT, Op0);
6645 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_r(RetVT, Op0);
6646 default: return Register();
6647 }
6648}
6649
6650// FastEmit functions for X86ISD::CVTTS2UIS_SAE.
6651
6652Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6653 if ((Subtarget->hasAVX10_2())) {
6654 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USISrrb_Int, RC: &X86::GR32RegClass, Op0);
6655 }
6656 return Register();
6657}
6658
6659Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6660 if ((Subtarget->hasAVX10_2())) {
6661 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6662 }
6663 return Register();
6664}
6665
6666Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6667switch (RetVT.SimpleTy) {
6668 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i32_r(Op0);
6669 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i64_r(Op0);
6670 default: return Register();
6671}
6672}
6673
6674Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6675 if ((Subtarget->hasAVX10_2())) {
6676 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USISrrb_Int, RC: &X86::GR32RegClass, Op0);
6677 }
6678 return Register();
6679}
6680
6681Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6682 if ((Subtarget->hasAVX10_2())) {
6683 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6684 }
6685 return Register();
6686}
6687
6688Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6689switch (RetVT.SimpleTy) {
6690 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i32_r(Op0);
6691 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i64_r(Op0);
6692 default: return Register();
6693}
6694}
6695
6696Register fastEmit_X86ISD_CVTTS2UIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6697 switch (VT.SimpleTy) {
6698 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_r(RetVT, Op0);
6699 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_r(RetVT, Op0);
6700 default: return Register();
6701 }
6702}
6703
6704// FastEmit functions for X86ISD::CVTTS2UI_SAE.
6705
6706Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Register Op0) {
6707 if ((Subtarget->hasFP16())) {
6708 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6709 }
6710 return Register();
6711}
6712
6713Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Register Op0) {
6714 if ((Subtarget->hasFP16())) {
6715 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6716 }
6717 return Register();
6718}
6719
6720Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6721switch (RetVT.SimpleTy) {
6722 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Op0);
6723 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Op0);
6724 default: return Register();
6725}
6726}
6727
6728Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6729 if ((Subtarget->hasAVX512())) {
6730 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6731 }
6732 return Register();
6733}
6734
6735Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6736 if ((Subtarget->hasAVX512())) {
6737 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6738 }
6739 return Register();
6740}
6741
6742Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6743switch (RetVT.SimpleTy) {
6744 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Op0);
6745 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Op0);
6746 default: return Register();
6747}
6748}
6749
6750Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6751 if ((Subtarget->hasAVX512())) {
6752 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6753 }
6754 return Register();
6755}
6756
6757Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6758 if ((Subtarget->hasAVX512())) {
6759 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6760 }
6761 return Register();
6762}
6763
6764Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6765switch (RetVT.SimpleTy) {
6766 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Op0);
6767 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Op0);
6768 default: return Register();
6769}
6770}
6771
6772Register fastEmit_X86ISD_CVTTS2UI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6773 switch (VT.SimpleTy) {
6774 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(RetVT, Op0);
6775 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(RetVT, Op0);
6776 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(RetVT, Op0);
6777 default: return Register();
6778 }
6779}
6780
6781// FastEmit functions for X86ISD::CVTUI2P.
6782
6783Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
6784 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6785 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6786 }
6787 return Register();
6788}
6789
6790Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
6791 if ((Subtarget->hasVLX())) {
6792 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
6793 }
6794 return Register();
6795}
6796
6797Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
6798switch (RetVT.SimpleTy) {
6799 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
6800 case MVT::v2f64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
6801 default: return Register();
6802}
6803}
6804
6805Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
6806 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6807 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6808 }
6809 return Register();
6810}
6811
6812Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
6813 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6814 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
6815 }
6816 return Register();
6817}
6818
6819Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
6820switch (RetVT.SimpleTy) {
6821 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
6822 case MVT::v4f32: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
6823 default: return Register();
6824}
6825}
6826
6827Register fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
6828 if (RetVT.SimpleTy != MVT::v8f16)
6829 return Register();
6830 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6831 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
6832 }
6833 return Register();
6834}
6835
6836Register fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, Register Op0) {
6837 switch (VT.SimpleTy) {
6838 case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
6839 case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
6840 case MVT::v4i64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
6841 default: return Register();
6842 }
6843}
6844
6845// FastEmit functions for X86ISD::FGETEXP.
6846
6847Register fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(MVT RetVT, Register Op0) {
6848 if (RetVT.SimpleTy != MVT::v8f16)
6849 return Register();
6850 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6851 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ128r, RC: &X86::VR128XRegClass, Op0);
6852 }
6853 return Register();
6854}
6855
6856Register fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(MVT RetVT, Register Op0) {
6857 if (RetVT.SimpleTy != MVT::v16f16)
6858 return Register();
6859 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6860 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ256r, RC: &X86::VR256XRegClass, Op0);
6861 }
6862 return Register();
6863}
6864
6865Register fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(MVT RetVT, Register Op0) {
6866 if (RetVT.SimpleTy != MVT::v32f16)
6867 return Register();
6868 if ((Subtarget->hasFP16())) {
6869 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZr, RC: &X86::VR512RegClass, Op0);
6870 }
6871 return Register();
6872}
6873
6874Register fastEmit_X86ISD_FGETEXP_MVT_v8bf16_r(MVT RetVT, Register Op0) {
6875 if (RetVT.SimpleTy != MVT::v8bf16)
6876 return Register();
6877 if ((Subtarget->hasAVX10_2())) {
6878 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Z128r, RC: &X86::VR128XRegClass, Op0);
6879 }
6880 return Register();
6881}
6882
6883Register fastEmit_X86ISD_FGETEXP_MVT_v16bf16_r(MVT RetVT, Register Op0) {
6884 if (RetVT.SimpleTy != MVT::v16bf16)
6885 return Register();
6886 if ((Subtarget->hasAVX10_2())) {
6887 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Z256r, RC: &X86::VR256XRegClass, Op0);
6888 }
6889 return Register();
6890}
6891
6892Register fastEmit_X86ISD_FGETEXP_MVT_v32bf16_r(MVT RetVT, Register Op0) {
6893 if (RetVT.SimpleTy != MVT::v32bf16)
6894 return Register();
6895 if ((Subtarget->hasAVX10_2())) {
6896 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Zr, RC: &X86::VR512RegClass, Op0);
6897 }
6898 return Register();
6899}
6900
6901Register fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(MVT RetVT, Register Op0) {
6902 if (RetVT.SimpleTy != MVT::v4f32)
6903 return Register();
6904 if ((Subtarget->hasVLX())) {
6905 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ128r, RC: &X86::VR128XRegClass, Op0);
6906 }
6907 return Register();
6908}
6909
6910Register fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(MVT RetVT, Register Op0) {
6911 if (RetVT.SimpleTy != MVT::v8f32)
6912 return Register();
6913 if ((Subtarget->hasVLX())) {
6914 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ256r, RC: &X86::VR256XRegClass, Op0);
6915 }
6916 return Register();
6917}
6918
6919Register fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(MVT RetVT, Register Op0) {
6920 if (RetVT.SimpleTy != MVT::v16f32)
6921 return Register();
6922 if ((Subtarget->hasAVX512())) {
6923 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZr, RC: &X86::VR512RegClass, Op0);
6924 }
6925 return Register();
6926}
6927
6928Register fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(MVT RetVT, Register Op0) {
6929 if (RetVT.SimpleTy != MVT::v2f64)
6930 return Register();
6931 if ((Subtarget->hasVLX())) {
6932 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ128r, RC: &X86::VR128XRegClass, Op0);
6933 }
6934 return Register();
6935}
6936
6937Register fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(MVT RetVT, Register Op0) {
6938 if (RetVT.SimpleTy != MVT::v4f64)
6939 return Register();
6940 if ((Subtarget->hasVLX())) {
6941 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ256r, RC: &X86::VR256XRegClass, Op0);
6942 }
6943 return Register();
6944}
6945
6946Register fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(MVT RetVT, Register Op0) {
6947 if (RetVT.SimpleTy != MVT::v8f64)
6948 return Register();
6949 if ((Subtarget->hasAVX512())) {
6950 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZr, RC: &X86::VR512RegClass, Op0);
6951 }
6952 return Register();
6953}
6954
6955Register fastEmit_X86ISD_FGETEXP_r(MVT VT, MVT RetVT, Register Op0) {
6956 switch (VT.SimpleTy) {
6957 case MVT::v8f16: return fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(RetVT, Op0);
6958 case MVT::v16f16: return fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(RetVT, Op0);
6959 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(RetVT, Op0);
6960 case MVT::v8bf16: return fastEmit_X86ISD_FGETEXP_MVT_v8bf16_r(RetVT, Op0);
6961 case MVT::v16bf16: return fastEmit_X86ISD_FGETEXP_MVT_v16bf16_r(RetVT, Op0);
6962 case MVT::v32bf16: return fastEmit_X86ISD_FGETEXP_MVT_v32bf16_r(RetVT, Op0);
6963 case MVT::v4f32: return fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(RetVT, Op0);
6964 case MVT::v8f32: return fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(RetVT, Op0);
6965 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(RetVT, Op0);
6966 case MVT::v2f64: return fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(RetVT, Op0);
6967 case MVT::v4f64: return fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(RetVT, Op0);
6968 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(RetVT, Op0);
6969 default: return Register();
6970 }
6971}
6972
6973// FastEmit functions for X86ISD::FGETEXP_SAE.
6974
6975Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
6976 if (RetVT.SimpleTy != MVT::v32f16)
6977 return Register();
6978 if ((Subtarget->hasFP16())) {
6979 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZrb, RC: &X86::VR512RegClass, Op0);
6980 }
6981 return Register();
6982}
6983
6984Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6985 if (RetVT.SimpleTy != MVT::v16f32)
6986 return Register();
6987 if ((Subtarget->hasAVX512())) {
6988 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZrb, RC: &X86::VR512RegClass, Op0);
6989 }
6990 return Register();
6991}
6992
6993Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6994 if (RetVT.SimpleTy != MVT::v8f64)
6995 return Register();
6996 if ((Subtarget->hasAVX512())) {
6997 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZrb, RC: &X86::VR512RegClass, Op0);
6998 }
6999 return Register();
7000}
7001
7002Register fastEmit_X86ISD_FGETEXP_SAE_r(MVT VT, MVT RetVT, Register Op0) {
7003 switch (VT.SimpleTy) {
7004 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(RetVT, Op0);
7005 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(RetVT, Op0);
7006 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(RetVT, Op0);
7007 default: return Register();
7008 }
7009}
7010
7011// FastEmit functions for X86ISD::FP_TO_SINT_SAT.
7012
7013Register fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7014 if (RetVT.SimpleTy != MVT::v2i64)
7015 return Register();
7016 if ((Subtarget->hasAVX10_2())) {
7017 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7018 }
7019 return Register();
7020}
7021
7022Register fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
7023 if (RetVT.SimpleTy != MVT::v4i32)
7024 return Register();
7025 if ((Subtarget->hasAVX10_2())) {
7026 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7027 }
7028 return Register();
7029}
7030
7031Register fastEmit_X86ISD_FP_TO_SINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
7032 switch (VT.SimpleTy) {
7033 case MVT::v4f32: return fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v4f32_r(RetVT, Op0);
7034 case MVT::v2f64: return fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v2f64_r(RetVT, Op0);
7035 default: return Register();
7036 }
7037}
7038
7039// FastEmit functions for X86ISD::FP_TO_UINT_SAT.
7040
7041Register fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7042 if (RetVT.SimpleTy != MVT::v2i64)
7043 return Register();
7044 if ((Subtarget->hasAVX10_2())) {
7045 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7046 }
7047 return Register();
7048}
7049
7050Register fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
7051 if (RetVT.SimpleTy != MVT::v4i32)
7052 return Register();
7053 if ((Subtarget->hasAVX10_2())) {
7054 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7055 }
7056 return Register();
7057}
7058
7059Register fastEmit_X86ISD_FP_TO_UINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
7060 switch (VT.SimpleTy) {
7061 case MVT::v4f32: return fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v4f32_r(RetVT, Op0);
7062 case MVT::v2f64: return fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v2f64_r(RetVT, Op0);
7063 default: return Register();
7064 }
7065}
7066
7067// FastEmit functions for X86ISD::FRCP.
7068
7069Register fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, Register Op0) {
7070 if (RetVT.SimpleTy != MVT::f32)
7071 return Register();
7072 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7073 return fastEmitInst_r(MachineInstOpcode: X86::RCPSSr, RC: &X86::FR32RegClass, Op0);
7074 }
7075 return Register();
7076}
7077
7078Register fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7079 if (RetVT.SimpleTy != MVT::v4f32)
7080 return Register();
7081 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7082 return fastEmitInst_r(MachineInstOpcode: X86::RCPPSr, RC: &X86::VR128RegClass, Op0);
7083 }
7084 if ((Subtarget->hasAVX())) {
7085 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSr, RC: &X86::VR128RegClass, Op0);
7086 }
7087 return Register();
7088}
7089
7090Register fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7091 if (RetVT.SimpleTy != MVT::v8f32)
7092 return Register();
7093 if ((Subtarget->hasAVX())) {
7094 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSYr, RC: &X86::VR256RegClass, Op0);
7095 }
7096 return Register();
7097}
7098
7099Register fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, Register Op0) {
7100 switch (VT.SimpleTy) {
7101 case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0);
7102 case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0);
7103 case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0);
7104 default: return Register();
7105 }
7106}
7107
7108// FastEmit functions for X86ISD::FRSQRT.
7109
7110Register fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
7111 if (RetVT.SimpleTy != MVT::f32)
7112 return Register();
7113 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7114 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTSSr, RC: &X86::FR32RegClass, Op0);
7115 }
7116 return Register();
7117}
7118
7119Register fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7120 if (RetVT.SimpleTy != MVT::v4f32)
7121 return Register();
7122 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7123 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTPSr, RC: &X86::VR128RegClass, Op0);
7124 }
7125 if ((Subtarget->hasAVX())) {
7126 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSr, RC: &X86::VR128RegClass, Op0);
7127 }
7128 return Register();
7129}
7130
7131Register fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
7132 if (RetVT.SimpleTy != MVT::v8f32)
7133 return Register();
7134 if ((Subtarget->hasAVX())) {
7135 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSYr, RC: &X86::VR256RegClass, Op0);
7136 }
7137 return Register();
7138}
7139
7140Register fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, Register Op0) {
7141 switch (VT.SimpleTy) {
7142 case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0);
7143 case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0);
7144 case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0);
7145 default: return Register();
7146 }
7147}
7148
7149// FastEmit functions for X86ISD::MMX_MOVD2W.
7150
7151Register fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, Register Op0) {
7152 if (RetVT.SimpleTy != MVT::i32)
7153 return Register();
7154 if ((Subtarget->hasMMX())) {
7155 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64grr, RC: &X86::GR32RegClass, Op0);
7156 }
7157 return Register();
7158}
7159
7160Register fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, Register Op0) {
7161 switch (VT.SimpleTy) {
7162 case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0);
7163 default: return Register();
7164 }
7165}
7166
7167// FastEmit functions for X86ISD::MMX_MOVW2D.
7168
7169Register fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, Register Op0) {
7170 if (RetVT.SimpleTy != MVT::x86mmx)
7171 return Register();
7172 if ((Subtarget->hasMMX())) {
7173 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64rr, RC: &X86::VR64RegClass, Op0);
7174 }
7175 return Register();
7176}
7177
7178Register fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, Register Op0) {
7179 switch (VT.SimpleTy) {
7180 case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0);
7181 default: return Register();
7182 }
7183}
7184
7185// FastEmit functions for X86ISD::MOVDDUP.
7186
7187Register fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, Register Op0) {
7188 if (RetVT.SimpleTy != MVT::v2f64)
7189 return Register();
7190 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7191 return fastEmitInst_r(MachineInstOpcode: X86::MOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7192 }
7193 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7194 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7195 }
7196 return Register();
7197}
7198
7199Register fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, Register Op0) {
7200 if (RetVT.SimpleTy != MVT::v4f64)
7201 return Register();
7202 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7203 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7204 }
7205 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7206 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPYrr, RC: &X86::VR256RegClass, Op0);
7207 }
7208 return Register();
7209}
7210
7211Register fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, Register Op0) {
7212 if (RetVT.SimpleTy != MVT::v8f64)
7213 return Register();
7214 if ((Subtarget->hasAVX512())) {
7215 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZrr, RC: &X86::VR512RegClass, Op0);
7216 }
7217 return Register();
7218}
7219
7220Register fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, Register Op0) {
7221 switch (VT.SimpleTy) {
7222 case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0);
7223 case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0);
7224 case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0);
7225 default: return Register();
7226 }
7227}
7228
7229// FastEmit functions for X86ISD::MOVDQ2Q.
7230
7231Register fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, Register Op0) {
7232 if (RetVT.SimpleTy != MVT::x86mmx)
7233 return Register();
7234 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
7235 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVDQ2Qrr, RC: &X86::VR64RegClass, Op0);
7236 }
7237 return Register();
7238}
7239
7240Register fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, Register Op0) {
7241 switch (VT.SimpleTy) {
7242 case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0);
7243 default: return Register();
7244 }
7245}
7246
7247// FastEmit functions for X86ISD::MOVMSK.
7248
7249Register fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, Register Op0) {
7250 if (RetVT.SimpleTy != MVT::i32)
7251 return Register();
7252 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7253 return fastEmitInst_r(MachineInstOpcode: X86::PMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
7254 }
7255 if ((Subtarget->hasAVX())) {
7256 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
7257 }
7258 return Register();
7259}
7260
7261Register fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, Register Op0) {
7262 if (RetVT.SimpleTy != MVT::i32)
7263 return Register();
7264 if ((Subtarget->hasAVX2())) {
7265 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBYrr, RC: &X86::GR32RegClass, Op0);
7266 }
7267 return Register();
7268}
7269
7270Register fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, Register Op0) {
7271 if (RetVT.SimpleTy != MVT::i32)
7272 return Register();
7273 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7274 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7275 }
7276 if ((Subtarget->hasAVX())) {
7277 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7278 }
7279 return Register();
7280}
7281
7282Register fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, Register Op0) {
7283 if (RetVT.SimpleTy != MVT::i32)
7284 return Register();
7285 if ((Subtarget->hasAVX())) {
7286 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
7287 }
7288 return Register();
7289}
7290
7291Register fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, Register Op0) {
7292 if (RetVT.SimpleTy != MVT::i32)
7293 return Register();
7294 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7295 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7296 }
7297 if ((Subtarget->hasAVX())) {
7298 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7299 }
7300 return Register();
7301}
7302
7303Register fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, Register Op0) {
7304 if (RetVT.SimpleTy != MVT::i32)
7305 return Register();
7306 if ((Subtarget->hasAVX())) {
7307 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
7308 }
7309 return Register();
7310}
7311
7312Register fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, Register Op0) {
7313 if (RetVT.SimpleTy != MVT::i32)
7314 return Register();
7315 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7316 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7317 }
7318 if ((Subtarget->hasAVX())) {
7319 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7320 }
7321 return Register();
7322}
7323
7324Register fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, Register Op0) {
7325 if (RetVT.SimpleTy != MVT::i32)
7326 return Register();
7327 if ((Subtarget->hasAVX())) {
7328 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
7329 }
7330 return Register();
7331}
7332
7333Register fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, Register Op0) {
7334 if (RetVT.SimpleTy != MVT::i32)
7335 return Register();
7336 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7337 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7338 }
7339 if ((Subtarget->hasAVX())) {
7340 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7341 }
7342 return Register();
7343}
7344
7345Register fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, Register Op0) {
7346 if (RetVT.SimpleTy != MVT::i32)
7347 return Register();
7348 if ((Subtarget->hasAVX())) {
7349 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
7350 }
7351 return Register();
7352}
7353
7354Register fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, Register Op0) {
7355 switch (VT.SimpleTy) {
7356 case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0);
7357 case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0);
7358 case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0);
7359 case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0);
7360 case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0);
7361 case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0);
7362 case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0);
7363 case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0);
7364 case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0);
7365 case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0);
7366 default: return Register();
7367 }
7368}
7369
7370// FastEmit functions for X86ISD::MOVQ2DQ.
7371
7372Register fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(MVT RetVT, Register Op0) {
7373 if (RetVT.SimpleTy != MVT::v2i64)
7374 return Register();
7375 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
7376 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2DQrr, RC: &X86::VR128RegClass, Op0);
7377 }
7378 return Register();
7379}
7380
7381Register fastEmit_X86ISD_MOVQ2DQ_r(MVT VT, MVT RetVT, Register Op0) {
7382 switch (VT.SimpleTy) {
7383 case MVT::x86mmx: return fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(RetVT, Op0);
7384 default: return Register();
7385 }
7386}
7387
7388// FastEmit functions for X86ISD::MOVSHDUP.
7389
7390Register fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, Register Op0) {
7391 if (RetVT.SimpleTy != MVT::v4i32)
7392 return Register();
7393 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7394 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7395 }
7396 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7397 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7398 }
7399 return Register();
7400}
7401
7402Register fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, Register Op0) {
7403 if (RetVT.SimpleTy != MVT::v8i32)
7404 return Register();
7405 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7406 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
7407 }
7408 return Register();
7409}
7410
7411Register fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7412 if (RetVT.SimpleTy != MVT::v4f32)
7413 return Register();
7414 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7415 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7416 }
7417 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7418 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7419 }
7420 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7421 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7422 }
7423 return Register();
7424}
7425
7426Register fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7427 if (RetVT.SimpleTy != MVT::v8f32)
7428 return Register();
7429 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7430 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7431 }
7432 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7433 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
7434 }
7435 return Register();
7436}
7437
7438Register fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7439 if (RetVT.SimpleTy != MVT::v16f32)
7440 return Register();
7441 if ((Subtarget->hasAVX512())) {
7442 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZrr, RC: &X86::VR512RegClass, Op0);
7443 }
7444 return Register();
7445}
7446
7447Register fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, Register Op0) {
7448 switch (VT.SimpleTy) {
7449 case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0);
7450 case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0);
7451 case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0);
7452 case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0);
7453 case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0);
7454 default: return Register();
7455 }
7456}
7457
7458// FastEmit functions for X86ISD::MOVSLDUP.
7459
7460Register fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, Register Op0) {
7461 if (RetVT.SimpleTy != MVT::v4i32)
7462 return Register();
7463 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7464 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7465 }
7466 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7467 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7468 }
7469 return Register();
7470}
7471
7472Register fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, Register Op0) {
7473 if (RetVT.SimpleTy != MVT::v8i32)
7474 return Register();
7475 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7476 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
7477 }
7478 return Register();
7479}
7480
7481Register fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7482 if (RetVT.SimpleTy != MVT::v4f32)
7483 return Register();
7484 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7485 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7486 }
7487 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7488 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7489 }
7490 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7491 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7492 }
7493 return Register();
7494}
7495
7496Register fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7497 if (RetVT.SimpleTy != MVT::v8f32)
7498 return Register();
7499 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7500 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7501 }
7502 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7503 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
7504 }
7505 return Register();
7506}
7507
7508Register fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7509 if (RetVT.SimpleTy != MVT::v16f32)
7510 return Register();
7511 if ((Subtarget->hasAVX512())) {
7512 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZrr, RC: &X86::VR512RegClass, Op0);
7513 }
7514 return Register();
7515}
7516
7517Register fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, Register Op0) {
7518 switch (VT.SimpleTy) {
7519 case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0);
7520 case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0);
7521 case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0);
7522 case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0);
7523 case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0);
7524 default: return Register();
7525 }
7526}
7527
7528// FastEmit functions for X86ISD::PHMINPOS.
7529
7530Register fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, Register Op0) {
7531 if (RetVT.SimpleTy != MVT::v8i16)
7532 return Register();
7533 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
7534 return fastEmitInst_r(MachineInstOpcode: X86::PHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
7535 }
7536 if ((Subtarget->hasAVX())) {
7537 return fastEmitInst_r(MachineInstOpcode: X86::VPHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
7538 }
7539 return Register();
7540}
7541
7542Register fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, Register Op0) {
7543 switch (VT.SimpleTy) {
7544 case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0);
7545 default: return Register();
7546 }
7547}
7548
7549// FastEmit functions for X86ISD::PROBED_ALLOCA.
7550
7551Register fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
7552 if (RetVT.SimpleTy != MVT::i32)
7553 return Register();
7554 if ((!Subtarget->isTarget64BitLP64())) {
7555 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
7556 }
7557 return Register();
7558}
7559
7560Register fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
7561 if (RetVT.SimpleTy != MVT::i64)
7562 return Register();
7563 if ((Subtarget->is64Bit())) {
7564 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
7565 }
7566 return Register();
7567}
7568
7569Register fastEmit_X86ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
7570 switch (VT.SimpleTy) {
7571 case MVT::i32: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(RetVT, Op0);
7572 case MVT::i64: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
7573 default: return Register();
7574 }
7575}
7576
7577// FastEmit functions for X86ISD::RCP14.
7578
7579Register fastEmit_X86ISD_RCP14_MVT_v8f16_r(MVT RetVT, Register Op0) {
7580 if (RetVT.SimpleTy != MVT::v8f16)
7581 return Register();
7582 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7583 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ128r, RC: &X86::VR128XRegClass, Op0);
7584 }
7585 return Register();
7586}
7587
7588Register fastEmit_X86ISD_RCP14_MVT_v16f16_r(MVT RetVT, Register Op0) {
7589 if (RetVT.SimpleTy != MVT::v16f16)
7590 return Register();
7591 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7592 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ256r, RC: &X86::VR256XRegClass, Op0);
7593 }
7594 return Register();
7595}
7596
7597Register fastEmit_X86ISD_RCP14_MVT_v32f16_r(MVT RetVT, Register Op0) {
7598 if (RetVT.SimpleTy != MVT::v32f16)
7599 return Register();
7600 if ((Subtarget->hasFP16())) {
7601 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZr, RC: &X86::VR512RegClass, Op0);
7602 }
7603 return Register();
7604}
7605
7606Register fastEmit_X86ISD_RCP14_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7607 if (RetVT.SimpleTy != MVT::v8bf16)
7608 return Register();
7609 if ((Subtarget->hasAVX10_2())) {
7610 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7611 }
7612 return Register();
7613}
7614
7615Register fastEmit_X86ISD_RCP14_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7616 if (RetVT.SimpleTy != MVT::v16bf16)
7617 return Register();
7618 if ((Subtarget->hasAVX10_2())) {
7619 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7620 }
7621 return Register();
7622}
7623
7624Register fastEmit_X86ISD_RCP14_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7625 if (RetVT.SimpleTy != MVT::v32bf16)
7626 return Register();
7627 if ((Subtarget->hasAVX10_2())) {
7628 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Zr, RC: &X86::VR512RegClass, Op0);
7629 }
7630 return Register();
7631}
7632
7633Register fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, Register Op0) {
7634 if (RetVT.SimpleTy != MVT::v4f32)
7635 return Register();
7636 if ((Subtarget->hasVLX())) {
7637 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ128r, RC: &X86::VR128XRegClass, Op0);
7638 }
7639 return Register();
7640}
7641
7642Register fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, Register Op0) {
7643 if (RetVT.SimpleTy != MVT::v8f32)
7644 return Register();
7645 if ((Subtarget->hasVLX())) {
7646 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ256r, RC: &X86::VR256XRegClass, Op0);
7647 }
7648 return Register();
7649}
7650
7651Register fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, Register Op0) {
7652 if (RetVT.SimpleTy != MVT::v16f32)
7653 return Register();
7654 if ((Subtarget->hasAVX512())) {
7655 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZr, RC: &X86::VR512RegClass, Op0);
7656 }
7657 return Register();
7658}
7659
7660Register fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, Register Op0) {
7661 if (RetVT.SimpleTy != MVT::v2f64)
7662 return Register();
7663 if ((Subtarget->hasVLX())) {
7664 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ128r, RC: &X86::VR128XRegClass, Op0);
7665 }
7666 return Register();
7667}
7668
7669Register fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, Register Op0) {
7670 if (RetVT.SimpleTy != MVT::v4f64)
7671 return Register();
7672 if ((Subtarget->hasVLX())) {
7673 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ256r, RC: &X86::VR256XRegClass, Op0);
7674 }
7675 return Register();
7676}
7677
7678Register fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, Register Op0) {
7679 if (RetVT.SimpleTy != MVT::v8f64)
7680 return Register();
7681 if ((Subtarget->hasAVX512())) {
7682 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZr, RC: &X86::VR512RegClass, Op0);
7683 }
7684 return Register();
7685}
7686
7687Register fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, Register Op0) {
7688 switch (VT.SimpleTy) {
7689 case MVT::v8f16: return fastEmit_X86ISD_RCP14_MVT_v8f16_r(RetVT, Op0);
7690 case MVT::v16f16: return fastEmit_X86ISD_RCP14_MVT_v16f16_r(RetVT, Op0);
7691 case MVT::v32f16: return fastEmit_X86ISD_RCP14_MVT_v32f16_r(RetVT, Op0);
7692 case MVT::v8bf16: return fastEmit_X86ISD_RCP14_MVT_v8bf16_r(RetVT, Op0);
7693 case MVT::v16bf16: return fastEmit_X86ISD_RCP14_MVT_v16bf16_r(RetVT, Op0);
7694 case MVT::v32bf16: return fastEmit_X86ISD_RCP14_MVT_v32bf16_r(RetVT, Op0);
7695 case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0);
7696 case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0);
7697 case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0);
7698 case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0);
7699 case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0);
7700 case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0);
7701 default: return Register();
7702 }
7703}
7704
7705// FastEmit functions for X86ISD::RSQRT14.
7706
7707Register fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(MVT RetVT, Register Op0) {
7708 if (RetVT.SimpleTy != MVT::v8f16)
7709 return Register();
7710 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7711 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
7712 }
7713 return Register();
7714}
7715
7716Register fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(MVT RetVT, Register Op0) {
7717 if (RetVT.SimpleTy != MVT::v16f16)
7718 return Register();
7719 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7720 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
7721 }
7722 return Register();
7723}
7724
7725Register fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(MVT RetVT, Register Op0) {
7726 if (RetVT.SimpleTy != MVT::v32f16)
7727 return Register();
7728 if ((Subtarget->hasFP16())) {
7729 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZr, RC: &X86::VR512RegClass, Op0);
7730 }
7731 return Register();
7732}
7733
7734Register fastEmit_X86ISD_RSQRT14_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7735 if (RetVT.SimpleTy != MVT::v8bf16)
7736 return Register();
7737 if ((Subtarget->hasAVX10_2())) {
7738 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7739 }
7740 return Register();
7741}
7742
7743Register fastEmit_X86ISD_RSQRT14_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7744 if (RetVT.SimpleTy != MVT::v16bf16)
7745 return Register();
7746 if ((Subtarget->hasAVX10_2())) {
7747 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7748 }
7749 return Register();
7750}
7751
7752Register fastEmit_X86ISD_RSQRT14_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7753 if (RetVT.SimpleTy != MVT::v32bf16)
7754 return Register();
7755 if ((Subtarget->hasAVX10_2())) {
7756 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
7757 }
7758 return Register();
7759}
7760
7761Register fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, Register Op0) {
7762 if (RetVT.SimpleTy != MVT::v4f32)
7763 return Register();
7764 if ((Subtarget->hasVLX())) {
7765 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ128r, RC: &X86::VR128XRegClass, Op0);
7766 }
7767 return Register();
7768}
7769
7770Register fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, Register Op0) {
7771 if (RetVT.SimpleTy != MVT::v8f32)
7772 return Register();
7773 if ((Subtarget->hasVLX())) {
7774 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ256r, RC: &X86::VR256XRegClass, Op0);
7775 }
7776 return Register();
7777}
7778
7779Register fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, Register Op0) {
7780 if (RetVT.SimpleTy != MVT::v16f32)
7781 return Register();
7782 if ((Subtarget->hasAVX512())) {
7783 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZr, RC: &X86::VR512RegClass, Op0);
7784 }
7785 return Register();
7786}
7787
7788Register fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, Register Op0) {
7789 if (RetVT.SimpleTy != MVT::v2f64)
7790 return Register();
7791 if ((Subtarget->hasVLX())) {
7792 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ128r, RC: &X86::VR128XRegClass, Op0);
7793 }
7794 return Register();
7795}
7796
7797Register fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, Register Op0) {
7798 if (RetVT.SimpleTy != MVT::v4f64)
7799 return Register();
7800 if ((Subtarget->hasVLX())) {
7801 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ256r, RC: &X86::VR256XRegClass, Op0);
7802 }
7803 return Register();
7804}
7805
7806Register fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, Register Op0) {
7807 if (RetVT.SimpleTy != MVT::v8f64)
7808 return Register();
7809 if ((Subtarget->hasAVX512())) {
7810 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZr, RC: &X86::VR512RegClass, Op0);
7811 }
7812 return Register();
7813}
7814
7815Register fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, Register Op0) {
7816 switch (VT.SimpleTy) {
7817 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(RetVT, Op0);
7818 case MVT::v16f16: return fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(RetVT, Op0);
7819 case MVT::v32f16: return fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(RetVT, Op0);
7820 case MVT::v8bf16: return fastEmit_X86ISD_RSQRT14_MVT_v8bf16_r(RetVT, Op0);
7821 case MVT::v16bf16: return fastEmit_X86ISD_RSQRT14_MVT_v16bf16_r(RetVT, Op0);
7822 case MVT::v32bf16: return fastEmit_X86ISD_RSQRT14_MVT_v32bf16_r(RetVT, Op0);
7823 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0);
7824 case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0);
7825 case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0);
7826 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0);
7827 case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0);
7828 case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0);
7829 default: return Register();
7830 }
7831}
7832
7833// FastEmit functions for X86ISD::SEG_ALLOCA.
7834
7835Register fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
7836 if (RetVT.SimpleTy != MVT::i32)
7837 return Register();
7838 if ((!Subtarget->isTarget64BitLP64())) {
7839 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
7840 }
7841 return Register();
7842}
7843
7844Register fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
7845 if (RetVT.SimpleTy != MVT::i64)
7846 return Register();
7847 if ((Subtarget->is64Bit())) {
7848 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
7849 }
7850 return Register();
7851}
7852
7853Register fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
7854 switch (VT.SimpleTy) {
7855 case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0);
7856 case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0);
7857 default: return Register();
7858 }
7859}
7860
7861// FastEmit functions for X86ISD::STRICT_CVTPH2PS.
7862
7863Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Register Op0) {
7864 if ((Subtarget->hasVLX())) {
7865 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
7866 }
7867 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
7868 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
7869 }
7870 return Register();
7871}
7872
7873Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Register Op0) {
7874 if ((Subtarget->hasVLX())) {
7875 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
7876 }
7877 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
7878 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
7879 }
7880 return Register();
7881}
7882
7883Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(MVT RetVT, Register Op0) {
7884switch (RetVT.SimpleTy) {
7885 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
7886 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
7887 default: return Register();
7888}
7889}
7890
7891Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(MVT RetVT, Register Op0) {
7892 if (RetVT.SimpleTy != MVT::v16f32)
7893 return Register();
7894 if ((Subtarget->hasAVX512())) {
7895 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
7896 }
7897 return Register();
7898}
7899
7900Register fastEmit_X86ISD_STRICT_CVTPH2PS_r(MVT VT, MVT RetVT, Register Op0) {
7901 switch (VT.SimpleTy) {
7902 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
7903 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
7904 default: return Register();
7905 }
7906}
7907
7908// FastEmit functions for X86ISD::STRICT_CVTSI2P.
7909
7910Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
7911 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7912 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
7913 }
7914 return Register();
7915}
7916
7917Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
7918 if ((Subtarget->hasVLX())) {
7919 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
7920 }
7921 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7922 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
7923 }
7924 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7925 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
7926 }
7927 return Register();
7928}
7929
7930Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
7931switch (RetVT.SimpleTy) {
7932 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
7933 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
7934 default: return Register();
7935}
7936}
7937
7938Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
7939 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7940 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
7941 }
7942 return Register();
7943}
7944
7945Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
7946 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7947 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
7948 }
7949 return Register();
7950}
7951
7952Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
7953switch (RetVT.SimpleTy) {
7954 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
7955 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
7956 default: return Register();
7957}
7958}
7959
7960Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
7961 if (RetVT.SimpleTy != MVT::v8f16)
7962 return Register();
7963 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7964 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
7965 }
7966 return Register();
7967}
7968
7969Register fastEmit_X86ISD_STRICT_CVTSI2P_r(MVT VT, MVT RetVT, Register Op0) {
7970 switch (VT.SimpleTy) {
7971 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
7972 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
7973 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
7974 default: return Register();
7975 }
7976}
7977
7978// FastEmit functions for X86ISD::STRICT_CVTTP2SI.
7979
7980Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
7981 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7982 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
7983 }
7984 return Register();
7985}
7986
7987Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
7988 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7989 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
7990 }
7991 return Register();
7992}
7993
7994Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
7995 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7996 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
7997 }
7998 return Register();
7999}
8000
8001Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
8002 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8003 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8004 }
8005 return Register();
8006}
8007
8008Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
8009 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8010 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8011 }
8012 return Register();
8013}
8014
8015Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
8016 if ((Subtarget->hasFP16())) {
8017 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
8018 }
8019 return Register();
8020}
8021
8022Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
8023switch (RetVT.SimpleTy) {
8024 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
8025 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
8026 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
8027 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
8028 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
8029 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
8030 default: return Register();
8031}
8032}
8033
8034Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
8035 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8036 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
8037 }
8038 return Register();
8039}
8040
8041Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
8042 if ((Subtarget->hasFP16())) {
8043 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
8044 }
8045 return Register();
8046}
8047
8048Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
8049switch (RetVT.SimpleTy) {
8050 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
8051 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
8052 default: return Register();
8053}
8054}
8055
8056Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
8057 if (RetVT.SimpleTy != MVT::v32i16)
8058 return Register();
8059 if ((Subtarget->hasFP16())) {
8060 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
8061 }
8062 return Register();
8063}
8064
8065Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
8066 if ((Subtarget->hasVLX())) {
8067 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8068 }
8069 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8070 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
8071 }
8072 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8073 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
8074 }
8075 return Register();
8076}
8077
8078Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
8079 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8080 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8081 }
8082 return Register();
8083}
8084
8085Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
8086 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8087 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8088 }
8089 return Register();
8090}
8091
8092Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
8093switch (RetVT.SimpleTy) {
8094 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
8095 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
8096 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
8097 default: return Register();
8098}
8099}
8100
8101Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
8102 if ((Subtarget->hasVLX())) {
8103 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
8104 }
8105 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8106 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
8107 }
8108 return Register();
8109}
8110
8111Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
8112 if ((Subtarget->hasDQI())) {
8113 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
8114 }
8115 return Register();
8116}
8117
8118Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
8119switch (RetVT.SimpleTy) {
8120 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
8121 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
8122 default: return Register();
8123}
8124}
8125
8126Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
8127 if (RetVT.SimpleTy != MVT::v16i32)
8128 return Register();
8129 if ((Subtarget->hasAVX512())) {
8130 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
8131 }
8132 return Register();
8133}
8134
8135Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
8136 if ((Subtarget->hasVLX())) {
8137 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8138 }
8139 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8140 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
8141 }
8142 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
8144 }
8145 return Register();
8146}
8147
8148Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
8149 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8150 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8151 }
8152 return Register();
8153}
8154
8155Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
8156switch (RetVT.SimpleTy) {
8157 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
8158 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
8159 default: return Register();
8160}
8161}
8162
8163Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
8164 if ((Subtarget->hasVLX())) {
8165 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
8166 }
8167 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8168 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
8169 }
8170 return Register();
8171}
8172
8173Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
8174 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8175 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8176 }
8177 return Register();
8178}
8179
8180Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
8181switch (RetVT.SimpleTy) {
8182 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
8183 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
8184 default: return Register();
8185}
8186}
8187
8188Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
8189 if ((Subtarget->hasAVX512())) {
8190 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
8191 }
8192 return Register();
8193}
8194
8195Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
8196 if ((Subtarget->hasDQI())) {
8197 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
8198 }
8199 return Register();
8200}
8201
8202Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
8203switch (RetVT.SimpleTy) {
8204 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
8205 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
8206 default: return Register();
8207}
8208}
8209
8210Register fastEmit_X86ISD_STRICT_CVTTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
8211 switch (VT.SimpleTy) {
8212 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
8213 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
8214 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
8215 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
8216 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
8217 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
8218 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
8219 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
8220 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
8221 default: return Register();
8222 }
8223}
8224
8225// FastEmit functions for X86ISD::STRICT_CVTTP2UI.
8226
8227Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
8228 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8229 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
8230 }
8231 return Register();
8232}
8233
8234Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
8235 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8236 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8237 }
8238 return Register();
8239}
8240
8241Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
8242 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8243 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
8244 }
8245 return Register();
8246}
8247
8248Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
8249 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8250 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8251 }
8252 return Register();
8253}
8254
8255Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
8256 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8257 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8258 }
8259 return Register();
8260}
8261
8262Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
8263 if ((Subtarget->hasFP16())) {
8264 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
8265 }
8266 return Register();
8267}
8268
8269Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
8270switch (RetVT.SimpleTy) {
8271 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
8272 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
8273 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
8274 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
8275 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
8276 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
8277 default: return Register();
8278}
8279}
8280
8281Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
8282 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8283 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
8284 }
8285 return Register();
8286}
8287
8288Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
8289 if ((Subtarget->hasFP16())) {
8290 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
8291 }
8292 return Register();
8293}
8294
8295Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
8296switch (RetVT.SimpleTy) {
8297 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
8298 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
8299 default: return Register();
8300}
8301}
8302
8303Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
8304 if (RetVT.SimpleTy != MVT::v32i16)
8305 return Register();
8306 if ((Subtarget->hasFP16())) {
8307 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
8308 }
8309 return Register();
8310}
8311
8312Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
8313 if ((Subtarget->hasVLX())) {
8314 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8315 }
8316 return Register();
8317}
8318
8319Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
8320 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8321 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8322 }
8323 return Register();
8324}
8325
8326Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
8327 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8328 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8329 }
8330 return Register();
8331}
8332
8333Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
8334switch (RetVT.SimpleTy) {
8335 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
8336 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
8337 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
8338 default: return Register();
8339}
8340}
8341
8342Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
8343 if ((Subtarget->hasVLX())) {
8344 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
8345 }
8346 return Register();
8347}
8348
8349Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
8350 if ((Subtarget->hasDQI())) {
8351 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
8352 }
8353 return Register();
8354}
8355
8356Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
8357switch (RetVT.SimpleTy) {
8358 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
8359 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
8360 default: return Register();
8361}
8362}
8363
8364Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
8365 if (RetVT.SimpleTy != MVT::v16i32)
8366 return Register();
8367 if ((Subtarget->hasAVX512())) {
8368 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
8369 }
8370 return Register();
8371}
8372
8373Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
8374 if ((Subtarget->hasVLX())) {
8375 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8376 }
8377 return Register();
8378}
8379
8380Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
8381 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8382 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8383 }
8384 return Register();
8385}
8386
8387Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
8388switch (RetVT.SimpleTy) {
8389 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
8390 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
8391 default: return Register();
8392}
8393}
8394
8395Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
8396 if ((Subtarget->hasVLX())) {
8397 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
8398 }
8399 return Register();
8400}
8401
8402Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
8403 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8404 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8405 }
8406 return Register();
8407}
8408
8409Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
8410switch (RetVT.SimpleTy) {
8411 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
8412 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
8413 default: return Register();
8414}
8415}
8416
8417Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
8418 if ((Subtarget->hasAVX512())) {
8419 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
8420 }
8421 return Register();
8422}
8423
8424Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
8425 if ((Subtarget->hasDQI())) {
8426 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
8427 }
8428 return Register();
8429}
8430
8431Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
8432switch (RetVT.SimpleTy) {
8433 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
8434 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
8435 default: return Register();
8436}
8437}
8438
8439Register fastEmit_X86ISD_STRICT_CVTTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
8440 switch (VT.SimpleTy) {
8441 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
8442 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
8443 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
8444 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
8445 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
8446 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
8447 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
8448 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
8449 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
8450 default: return Register();
8451 }
8452}
8453
8454// FastEmit functions for X86ISD::STRICT_CVTUI2P.
8455
8456Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
8457 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8458 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8459 }
8460 return Register();
8461}
8462
8463Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
8464 if ((Subtarget->hasVLX())) {
8465 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8466 }
8467 return Register();
8468}
8469
8470Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
8471switch (RetVT.SimpleTy) {
8472 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
8473 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
8474 default: return Register();
8475}
8476}
8477
8478Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
8479 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8481 }
8482 return Register();
8483}
8484
8485Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
8486 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8487 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8488 }
8489 return Register();
8490}
8491
8492Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
8493switch (RetVT.SimpleTy) {
8494 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
8495 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
8496 default: return Register();
8497}
8498}
8499
8500Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
8501 if (RetVT.SimpleTy != MVT::v8f16)
8502 return Register();
8503 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8504 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8505 }
8506 return Register();
8507}
8508
8509Register fastEmit_X86ISD_STRICT_CVTUI2P_r(MVT VT, MVT RetVT, Register Op0) {
8510 switch (VT.SimpleTy) {
8511 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
8512 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
8513 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
8514 default: return Register();
8515 }
8516}
8517
8518// FastEmit functions for X86ISD::STRICT_VFPEXT.
8519
8520Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Register Op0) {
8521 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8522 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
8523 }
8524 return Register();
8525}
8526
8527Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Register Op0) {
8528 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8529 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8530 }
8531 return Register();
8532}
8533
8534Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Register Op0) {
8535 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8536 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
8537 }
8538 return Register();
8539}
8540
8541Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(MVT RetVT, Register Op0) {
8542switch (RetVT.SimpleTy) {
8543 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
8544 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
8545 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
8546 default: return Register();
8547}
8548}
8549
8550Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(MVT RetVT, Register Op0) {
8551 if (RetVT.SimpleTy != MVT::v2f64)
8552 return Register();
8553 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8554 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8555 }
8556 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8557 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
8558 }
8559 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8560 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
8561 }
8562 return Register();
8563}
8564
8565Register fastEmit_X86ISD_STRICT_VFPEXT_r(MVT VT, MVT RetVT, Register Op0) {
8566 switch (VT.SimpleTy) {
8567 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(RetVT, Op0);
8568 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(RetVT, Op0);
8569 default: return Register();
8570 }
8571}
8572
8573// FastEmit functions for X86ISD::STRICT_VFPROUND.
8574
8575Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
8576 if (RetVT.SimpleTy != MVT::v8f16)
8577 return Register();
8578 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8579 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
8580 }
8581 return Register();
8582}
8583
8584Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(MVT RetVT, Register Op0) {
8585 if (RetVT.SimpleTy != MVT::v8f16)
8586 return Register();
8587 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8588 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
8589 }
8590 return Register();
8591}
8592
8593Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(MVT RetVT, Register Op0) {
8594 if (RetVT.SimpleTy != MVT::v16f16)
8595 return Register();
8596 if ((Subtarget->hasFP16())) {
8597 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
8598 }
8599 return Register();
8600}
8601
8602Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Register Op0) {
8603 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8604 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8605 }
8606 return Register();
8607}
8608
8609Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Register Op0) {
8610 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8611 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8612 }
8613 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8614 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8615 }
8616 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8617 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8618 }
8619 return Register();
8620}
8621
8622Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
8623switch (RetVT.SimpleTy) {
8624 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
8625 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
8626 default: return Register();
8627}
8628}
8629
8630Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Register Op0) {
8631 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8632 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8633 }
8634 return Register();
8635}
8636
8637Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Register Op0) {
8638 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8639 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
8640 }
8641 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8642 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
8643 }
8644 return Register();
8645}
8646
8647Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(MVT RetVT, Register Op0) {
8648switch (RetVT.SimpleTy) {
8649 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
8650 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
8651 default: return Register();
8652}
8653}
8654
8655Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Register Op0) {
8656 if ((Subtarget->hasFP16())) {
8657 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
8658 }
8659 return Register();
8660}
8661
8662Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Register Op0) {
8663 if ((Subtarget->hasAVX512())) {
8664 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
8665 }
8666 return Register();
8667}
8668
8669Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(MVT RetVT, Register Op0) {
8670switch (RetVT.SimpleTy) {
8671 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
8672 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
8673 default: return Register();
8674}
8675}
8676
8677Register fastEmit_X86ISD_STRICT_VFPROUND_r(MVT VT, MVT RetVT, Register Op0) {
8678 switch (VT.SimpleTy) {
8679 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(RetVT, Op0);
8680 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(RetVT, Op0);
8681 case MVT::v16f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(RetVT, Op0);
8682 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(RetVT, Op0);
8683 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(RetVT, Op0);
8684 case MVT::v8f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(RetVT, Op0);
8685 default: return Register();
8686 }
8687}
8688
8689// FastEmit functions for X86ISD::VBROADCAST.
8690
8691Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Register Op0) {
8692 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8693 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ128rr, RC: &X86::VR128XRegClass, Op0);
8694 }
8695 return Register();
8696}
8697
8698Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Register Op0) {
8699 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8700 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ256rr, RC: &X86::VR256XRegClass, Op0);
8701 }
8702 return Register();
8703}
8704
8705Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Register Op0) {
8706 if ((Subtarget->hasAVX512())) {
8707 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZrr, RC: &X86::VR512RegClass, Op0);
8708 }
8709 return Register();
8710}
8711
8712Register fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, Register Op0) {
8713switch (RetVT.SimpleTy) {
8714 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0);
8715 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0);
8716 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0);
8717 default: return Register();
8718}
8719}
8720
8721Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Register Op0) {
8722 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8723 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ128rr, RC: &X86::VR128XRegClass, Op0);
8724 }
8725 return Register();
8726}
8727
8728Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Register Op0) {
8729 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8730 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ256rr, RC: &X86::VR256XRegClass, Op0);
8731 }
8732 return Register();
8733}
8734
8735Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Register Op0) {
8736 if ((Subtarget->hasAVX512())) {
8737 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZrr, RC: &X86::VR512RegClass, Op0);
8738 }
8739 return Register();
8740}
8741
8742Register fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, Register Op0) {
8743switch (RetVT.SimpleTy) {
8744 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0);
8745 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0);
8746 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0);
8747 default: return Register();
8748}
8749}
8750
8751Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Register Op0) {
8752 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8753 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ128rr, RC: &X86::VR128XRegClass, Op0);
8754 }
8755 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8756 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBrr, RC: &X86::VR128RegClass, Op0);
8757 }
8758 return Register();
8759}
8760
8761Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Register Op0) {
8762 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8763 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ256rr, RC: &X86::VR256XRegClass, Op0);
8764 }
8765 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8766 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBYrr, RC: &X86::VR256RegClass, Op0);
8767 }
8768 return Register();
8769}
8770
8771Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Register Op0) {
8772 if ((Subtarget->hasBWI())) {
8773 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZrr, RC: &X86::VR512RegClass, Op0);
8774 }
8775 return Register();
8776}
8777
8778Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
8779switch (RetVT.SimpleTy) {
8780 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0);
8781 case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0);
8782 case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0);
8783 default: return Register();
8784}
8785}
8786
8787Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Register Op0) {
8788 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8789 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
8790 }
8791 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8792 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
8793 }
8794 return Register();
8795}
8796
8797Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Register Op0) {
8798 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8799 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
8800 }
8801 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8802 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
8803 }
8804 return Register();
8805}
8806
8807Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Register Op0) {
8808 if ((Subtarget->hasBWI())) {
8809 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
8810 }
8811 return Register();
8812}
8813
8814Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
8815switch (RetVT.SimpleTy) {
8816 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0);
8817 case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0);
8818 case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0);
8819 default: return Register();
8820}
8821}
8822
8823Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Register Op0) {
8824 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8825 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ128rr, RC: &X86::VR128XRegClass, Op0);
8826 }
8827 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8828 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrr, RC: &X86::VR128RegClass, Op0);
8829 }
8830 return Register();
8831}
8832
8833Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Register Op0) {
8834 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8835 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ256rr, RC: &X86::VR256XRegClass, Op0);
8836 }
8837 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8838 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDYrr, RC: &X86::VR256RegClass, Op0);
8839 }
8840 return Register();
8841}
8842
8843Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Register Op0) {
8844 if ((Subtarget->hasAVX512())) {
8845 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZrr, RC: &X86::VR512RegClass, Op0);
8846 }
8847 return Register();
8848}
8849
8850Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
8851switch (RetVT.SimpleTy) {
8852 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0);
8853 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0);
8854 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0);
8855 default: return Register();
8856}
8857}
8858
8859Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Register Op0) {
8860 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8861 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ128rr, RC: &X86::VR128XRegClass, Op0);
8862 }
8863 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8864 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrr, RC: &X86::VR128RegClass, Op0);
8865 }
8866 return Register();
8867}
8868
8869Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Register Op0) {
8870 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8871 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ256rr, RC: &X86::VR256XRegClass, Op0);
8872 }
8873 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8874 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQYrr, RC: &X86::VR256RegClass, Op0);
8875 }
8876 return Register();
8877}
8878
8879Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Register Op0) {
8880 if ((Subtarget->hasAVX512())) {
8881 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZrr, RC: &X86::VR512RegClass, Op0);
8882 }
8883 return Register();
8884}
8885
8886Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
8887switch (RetVT.SimpleTy) {
8888 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0);
8889 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0);
8890 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0);
8891 default: return Register();
8892}
8893}
8894
8895Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Register Op0) {
8896 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8897 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
8898 }
8899 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8900 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
8901 }
8902 return Register();
8903}
8904
8905Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Register Op0) {
8906 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8907 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
8908 }
8909 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8910 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
8911 }
8912 return Register();
8913}
8914
8915Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Register Op0) {
8916 if ((Subtarget->hasBWI())) {
8917 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
8918 }
8919 return Register();
8920}
8921
8922Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
8923switch (RetVT.SimpleTy) {
8924 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Op0);
8925 case MVT::v16f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Op0);
8926 case MVT::v32f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Op0);
8927 default: return Register();
8928}
8929}
8930
8931Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Register Op0) {
8932 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
8933 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
8934 }
8935 return Register();
8936}
8937
8938Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Register Op0) {
8939 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
8940 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
8941 }
8942 return Register();
8943}
8944
8945Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Register Op0) {
8946 if ((Subtarget->hasBF16())) {
8947 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
8948 }
8949 return Register();
8950}
8951
8952Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
8953switch (RetVT.SimpleTy) {
8954 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Op0);
8955 case MVT::v16bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Op0);
8956 case MVT::v32bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Op0);
8957 default: return Register();
8958}
8959}
8960
8961Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Register Op0) {
8962 if ((Subtarget->hasVLX())) {
8963 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ128rr, RC: &X86::VR128XRegClass, Op0);
8964 }
8965 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8966 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSrr, RC: &X86::VR128RegClass, Op0);
8967 }
8968 return Register();
8969}
8970
8971Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Register Op0) {
8972 if ((Subtarget->hasVLX())) {
8973 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ256rr, RC: &X86::VR256XRegClass, Op0);
8974 }
8975 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8976 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSYrr, RC: &X86::VR256RegClass, Op0);
8977 }
8978 return Register();
8979}
8980
8981Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Register Op0) {
8982 if ((Subtarget->hasAVX512())) {
8983 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZrr, RC: &X86::VR512RegClass, Op0);
8984 }
8985 return Register();
8986}
8987
8988Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
8989switch (RetVT.SimpleTy) {
8990 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0);
8991 case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0);
8992 case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0);
8993 default: return Register();
8994}
8995}
8996
8997Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Register Op0) {
8998 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8999 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
9000 }
9001 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9002 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
9003 }
9004 return Register();
9005}
9006
9007Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Register Op0) {
9008 if ((Subtarget->hasVLX())) {
9009 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZ256rr, RC: &X86::VR256XRegClass, Op0);
9010 }
9011 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9012 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDYrr, RC: &X86::VR256RegClass, Op0);
9013 }
9014 return Register();
9015}
9016
9017Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Register Op0) {
9018 if ((Subtarget->hasAVX512())) {
9019 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZrr, RC: &X86::VR512RegClass, Op0);
9020 }
9021 return Register();
9022}
9023
9024Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
9025switch (RetVT.SimpleTy) {
9026 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0);
9027 case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0);
9028 case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0);
9029 default: return Register();
9030}
9031}
9032
9033Register fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, Register Op0) {
9034 switch (VT.SimpleTy) {
9035 case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0);
9036 case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0);
9037 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0);
9038 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0);
9039 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0);
9040 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0);
9041 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(RetVT, Op0);
9042 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(RetVT, Op0);
9043 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0);
9044 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0);
9045 default: return Register();
9046 }
9047}
9048
9049// FastEmit functions for X86ISD::VBROADCASTM.
9050
9051Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Register Op0) {
9052 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9053 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ128rr, RC: &X86::VR128XRegClass, Op0);
9054 }
9055 return Register();
9056}
9057
9058Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Register Op0) {
9059 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9060 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ256rr, RC: &X86::VR256XRegClass, Op0);
9061 }
9062 return Register();
9063}
9064
9065Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Register Op0) {
9066 if ((Subtarget->hasCDI())) {
9067 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZrr, RC: &X86::VR512RegClass, Op0);
9068 }
9069 return Register();
9070}
9071
9072Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, Register Op0) {
9073switch (RetVT.SimpleTy) {
9074 case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0);
9075 case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0);
9076 case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0);
9077 default: return Register();
9078}
9079}
9080
9081Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Register Op0) {
9082 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9083 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ128rr, RC: &X86::VR128XRegClass, Op0);
9084 }
9085 return Register();
9086}
9087
9088Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Register Op0) {
9089 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9090 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ256rr, RC: &X86::VR256XRegClass, Op0);
9091 }
9092 return Register();
9093}
9094
9095Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Register Op0) {
9096 if ((Subtarget->hasCDI())) {
9097 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZrr, RC: &X86::VR512RegClass, Op0);
9098 }
9099 return Register();
9100}
9101
9102Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, Register Op0) {
9103switch (RetVT.SimpleTy) {
9104 case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0);
9105 case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0);
9106 case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0);
9107 default: return Register();
9108}
9109}
9110
9111Register fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, Register Op0) {
9112 switch (VT.SimpleTy) {
9113 case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0);
9114 case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0);
9115 default: return Register();
9116 }
9117}
9118
9119// FastEmit functions for X86ISD::VCVTBF82PS.
9120
9121Register fastEmit_X86ISD_VCVTBF82PS_MVT_v16i8_MVT_v4f32_r(Register Op0) {
9122 if ((Subtarget->hasAVX10_V2_AUX())) {
9123 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF82PSZ128rr, RC: &X86::VR128XRegClass, Op0);
9124 }
9125 return Register();
9126}
9127
9128Register fastEmit_X86ISD_VCVTBF82PS_MVT_v16i8_MVT_v8f32_r(Register Op0) {
9129 if ((Subtarget->hasAVX10_V2_AUX())) {
9130 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF82PSZ256rr, RC: &X86::VR256XRegClass, Op0);
9131 }
9132 return Register();
9133}
9134
9135Register fastEmit_X86ISD_VCVTBF82PS_MVT_v16i8_MVT_v16f32_r(Register Op0) {
9136 if ((Subtarget->hasAVX10_V2_AUX())) {
9137 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF82PSZrr, RC: &X86::VR512RegClass, Op0);
9138 }
9139 return Register();
9140}
9141
9142Register fastEmit_X86ISD_VCVTBF82PS_MVT_v16i8_r(MVT RetVT, Register Op0) {
9143switch (RetVT.SimpleTy) {
9144 case MVT::v4f32: return fastEmit_X86ISD_VCVTBF82PS_MVT_v16i8_MVT_v4f32_r(Op0);
9145 case MVT::v8f32: return fastEmit_X86ISD_VCVTBF82PS_MVT_v16i8_MVT_v8f32_r(Op0);
9146 case MVT::v16f32: return fastEmit_X86ISD_VCVTBF82PS_MVT_v16i8_MVT_v16f32_r(Op0);
9147 default: return Register();
9148}
9149}
9150
9151Register fastEmit_X86ISD_VCVTBF82PS_r(MVT VT, MVT RetVT, Register Op0) {
9152 switch (VT.SimpleTy) {
9153 case MVT::v16i8: return fastEmit_X86ISD_VCVTBF82PS_MVT_v16i8_r(RetVT, Op0);
9154 default: return Register();
9155 }
9156}
9157
9158// FastEmit functions for X86ISD::VCVTHF82PH.
9159
9160Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v8f16_r(Register Op0) {
9161 if ((Subtarget->hasAVX10_2())) {
9162 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZ128rr, RC: &X86::VR128XRegClass, Op0);
9163 }
9164 return Register();
9165}
9166
9167Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v16f16_r(Register Op0) {
9168 if ((Subtarget->hasAVX10_2())) {
9169 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZ256rr, RC: &X86::VR256XRegClass, Op0);
9170 }
9171 return Register();
9172}
9173
9174Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_r(MVT RetVT, Register Op0) {
9175switch (RetVT.SimpleTy) {
9176 case MVT::v8f16: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v8f16_r(Op0);
9177 case MVT::v16f16: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v16f16_r(Op0);
9178 default: return Register();
9179}
9180}
9181
9182Register fastEmit_X86ISD_VCVTHF82PH_MVT_v32i8_r(MVT RetVT, Register Op0) {
9183 if (RetVT.SimpleTy != MVT::v32f16)
9184 return Register();
9185 if ((Subtarget->hasAVX10_2())) {
9186 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZrr, RC: &X86::VR512RegClass, Op0);
9187 }
9188 return Register();
9189}
9190
9191Register fastEmit_X86ISD_VCVTHF82PH_r(MVT VT, MVT RetVT, Register Op0) {
9192 switch (VT.SimpleTy) {
9193 case MVT::v16i8: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_r(RetVT, Op0);
9194 case MVT::v32i8: return fastEmit_X86ISD_VCVTHF82PH_MVT_v32i8_r(RetVT, Op0);
9195 default: return Register();
9196 }
9197}
9198
9199// FastEmit functions for X86ISD::VCVTHF82PS.
9200
9201Register fastEmit_X86ISD_VCVTHF82PS_MVT_v16i8_MVT_v4f32_r(Register Op0) {
9202 if ((Subtarget->hasAVX10_V2_AUX())) {
9203 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PSZ128rr, RC: &X86::VR128XRegClass, Op0);
9204 }
9205 return Register();
9206}
9207
9208Register fastEmit_X86ISD_VCVTHF82PS_MVT_v16i8_MVT_v8f32_r(Register Op0) {
9209 if ((Subtarget->hasAVX10_V2_AUX())) {
9210 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PSZ256rr, RC: &X86::VR256XRegClass, Op0);
9211 }
9212 return Register();
9213}
9214
9215Register fastEmit_X86ISD_VCVTHF82PS_MVT_v16i8_MVT_v16f32_r(Register Op0) {
9216 if ((Subtarget->hasAVX10_V2_AUX())) {
9217 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PSZrr, RC: &X86::VR512RegClass, Op0);
9218 }
9219 return Register();
9220}
9221
9222Register fastEmit_X86ISD_VCVTHF82PS_MVT_v16i8_r(MVT RetVT, Register Op0) {
9223switch (RetVT.SimpleTy) {
9224 case MVT::v4f32: return fastEmit_X86ISD_VCVTHF82PS_MVT_v16i8_MVT_v4f32_r(Op0);
9225 case MVT::v8f32: return fastEmit_X86ISD_VCVTHF82PS_MVT_v16i8_MVT_v8f32_r(Op0);
9226 case MVT::v16f32: return fastEmit_X86ISD_VCVTHF82PS_MVT_v16i8_MVT_v16f32_r(Op0);
9227 default: return Register();
9228}
9229}
9230
9231Register fastEmit_X86ISD_VCVTHF82PS_r(MVT VT, MVT RetVT, Register Op0) {
9232 switch (VT.SimpleTy) {
9233 case MVT::v16i8: return fastEmit_X86ISD_VCVTHF82PS_MVT_v16i8_r(RetVT, Op0);
9234 default: return Register();
9235 }
9236}
9237
9238// FastEmit functions for X86ISD::VCVTPH2BF8.
9239
9240Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v8f16_r(MVT RetVT, Register Op0) {
9241 if (RetVT.SimpleTy != MVT::v16i8)
9242 return Register();
9243 if ((Subtarget->hasAVX10_2())) {
9244 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9245 }
9246 return Register();
9247}
9248
9249Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v16f16_r(MVT RetVT, Register Op0) {
9250 if (RetVT.SimpleTy != MVT::v16i8)
9251 return Register();
9252 if ((Subtarget->hasAVX10_2())) {
9253 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9254 }
9255 return Register();
9256}
9257
9258Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v32f16_r(MVT RetVT, Register Op0) {
9259 if (RetVT.SimpleTy != MVT::v32i8)
9260 return Register();
9261 if ((Subtarget->hasAVX10_2())) {
9262 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Zrr, RC: &X86::VR256XRegClass, Op0);
9263 }
9264 return Register();
9265}
9266
9267Register fastEmit_X86ISD_VCVTPH2BF8_r(MVT VT, MVT RetVT, Register Op0) {
9268 switch (VT.SimpleTy) {
9269 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v8f16_r(RetVT, Op0);
9270 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v16f16_r(RetVT, Op0);
9271 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v32f16_r(RetVT, Op0);
9272 default: return Register();
9273 }
9274}
9275
9276// FastEmit functions for X86ISD::VCVTPH2BF8S.
9277
9278Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v8f16_r(MVT RetVT, Register Op0) {
9279 if (RetVT.SimpleTy != MVT::v16i8)
9280 return Register();
9281 if ((Subtarget->hasAVX10_2())) {
9282 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9283 }
9284 return Register();
9285}
9286
9287Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v16f16_r(MVT RetVT, Register Op0) {
9288 if (RetVT.SimpleTy != MVT::v16i8)
9289 return Register();
9290 if ((Subtarget->hasAVX10_2())) {
9291 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9292 }
9293 return Register();
9294}
9295
9296Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v32f16_r(MVT RetVT, Register Op0) {
9297 if (RetVT.SimpleTy != MVT::v32i8)
9298 return Register();
9299 if ((Subtarget->hasAVX10_2())) {
9300 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZrr, RC: &X86::VR256XRegClass, Op0);
9301 }
9302 return Register();
9303}
9304
9305Register fastEmit_X86ISD_VCVTPH2BF8S_r(MVT VT, MVT RetVT, Register Op0) {
9306 switch (VT.SimpleTy) {
9307 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v8f16_r(RetVT, Op0);
9308 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v16f16_r(RetVT, Op0);
9309 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v32f16_r(RetVT, Op0);
9310 default: return Register();
9311 }
9312}
9313
9314// FastEmit functions for X86ISD::VCVTPH2HF8.
9315
9316Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v8f16_r(MVT RetVT, Register Op0) {
9317 if (RetVT.SimpleTy != MVT::v16i8)
9318 return Register();
9319 if ((Subtarget->hasAVX10_2())) {
9320 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9321 }
9322 return Register();
9323}
9324
9325Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v16f16_r(MVT RetVT, Register Op0) {
9326 if (RetVT.SimpleTy != MVT::v16i8)
9327 return Register();
9328 if ((Subtarget->hasAVX10_2())) {
9329 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9330 }
9331 return Register();
9332}
9333
9334Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v32f16_r(MVT RetVT, Register Op0) {
9335 if (RetVT.SimpleTy != MVT::v32i8)
9336 return Register();
9337 if ((Subtarget->hasAVX10_2())) {
9338 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Zrr, RC: &X86::VR256XRegClass, Op0);
9339 }
9340 return Register();
9341}
9342
9343Register fastEmit_X86ISD_VCVTPH2HF8_r(MVT VT, MVT RetVT, Register Op0) {
9344 switch (VT.SimpleTy) {
9345 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v8f16_r(RetVT, Op0);
9346 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v16f16_r(RetVT, Op0);
9347 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v32f16_r(RetVT, Op0);
9348 default: return Register();
9349 }
9350}
9351
9352// FastEmit functions for X86ISD::VCVTPH2HF8S.
9353
9354Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v8f16_r(MVT RetVT, Register Op0) {
9355 if (RetVT.SimpleTy != MVT::v16i8)
9356 return Register();
9357 if ((Subtarget->hasAVX10_2())) {
9358 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9359 }
9360 return Register();
9361}
9362
9363Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v16f16_r(MVT RetVT, Register Op0) {
9364 if (RetVT.SimpleTy != MVT::v16i8)
9365 return Register();
9366 if ((Subtarget->hasAVX10_2())) {
9367 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9368 }
9369 return Register();
9370}
9371
9372Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v32f16_r(MVT RetVT, Register Op0) {
9373 if (RetVT.SimpleTy != MVT::v32i8)
9374 return Register();
9375 if ((Subtarget->hasAVX10_2())) {
9376 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZrr, RC: &X86::VR256XRegClass, Op0);
9377 }
9378 return Register();
9379}
9380
9381Register fastEmit_X86ISD_VCVTPH2HF8S_r(MVT VT, MVT RetVT, Register Op0) {
9382 switch (VT.SimpleTy) {
9383 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v8f16_r(RetVT, Op0);
9384 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v16f16_r(RetVT, Op0);
9385 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v32f16_r(RetVT, Op0);
9386 default: return Register();
9387 }
9388}
9389
9390// FastEmit functions for X86ISD::VCVTPS2BF8.
9391
9392Register fastEmit_X86ISD_VCVTPS2BF8_MVT_v4f32_r(MVT RetVT, Register Op0) {
9393 if (RetVT.SimpleTy != MVT::v16i8)
9394 return Register();
9395 if ((Subtarget->hasAVX10_V2_AUX())) {
9396 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2BF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9397 }
9398 return Register();
9399}
9400
9401Register fastEmit_X86ISD_VCVTPS2BF8_MVT_v8f32_r(MVT RetVT, Register Op0) {
9402 if (RetVT.SimpleTy != MVT::v16i8)
9403 return Register();
9404 if ((Subtarget->hasAVX10_V2_AUX())) {
9405 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2BF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9406 }
9407 return Register();
9408}
9409
9410Register fastEmit_X86ISD_VCVTPS2BF8_MVT_v16f32_r(MVT RetVT, Register Op0) {
9411 if (RetVT.SimpleTy != MVT::v16i8)
9412 return Register();
9413 if ((Subtarget->hasAVX10_V2_AUX())) {
9414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2BF8Zrr, RC: &X86::VR128XRegClass, Op0);
9415 }
9416 return Register();
9417}
9418
9419Register fastEmit_X86ISD_VCVTPS2BF8_r(MVT VT, MVT RetVT, Register Op0) {
9420 switch (VT.SimpleTy) {
9421 case MVT::v4f32: return fastEmit_X86ISD_VCVTPS2BF8_MVT_v4f32_r(RetVT, Op0);
9422 case MVT::v8f32: return fastEmit_X86ISD_VCVTPS2BF8_MVT_v8f32_r(RetVT, Op0);
9423 case MVT::v16f32: return fastEmit_X86ISD_VCVTPS2BF8_MVT_v16f32_r(RetVT, Op0);
9424 default: return Register();
9425 }
9426}
9427
9428// FastEmit functions for X86ISD::VCVTPS2BF8S.
9429
9430Register fastEmit_X86ISD_VCVTPS2BF8S_MVT_v4f32_r(MVT RetVT, Register Op0) {
9431 if (RetVT.SimpleTy != MVT::v16i8)
9432 return Register();
9433 if ((Subtarget->hasAVX10_V2_AUX())) {
9434 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9435 }
9436 return Register();
9437}
9438
9439Register fastEmit_X86ISD_VCVTPS2BF8S_MVT_v8f32_r(MVT RetVT, Register Op0) {
9440 if (RetVT.SimpleTy != MVT::v16i8)
9441 return Register();
9442 if ((Subtarget->hasAVX10_V2_AUX())) {
9443 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2BF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9444 }
9445 return Register();
9446}
9447
9448Register fastEmit_X86ISD_VCVTPS2BF8S_MVT_v16f32_r(MVT RetVT, Register Op0) {
9449 if (RetVT.SimpleTy != MVT::v16i8)
9450 return Register();
9451 if ((Subtarget->hasAVX10_V2_AUX())) {
9452 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2BF8SZrr, RC: &X86::VR128XRegClass, Op0);
9453 }
9454 return Register();
9455}
9456
9457Register fastEmit_X86ISD_VCVTPS2BF8S_r(MVT VT, MVT RetVT, Register Op0) {
9458 switch (VT.SimpleTy) {
9459 case MVT::v4f32: return fastEmit_X86ISD_VCVTPS2BF8S_MVT_v4f32_r(RetVT, Op0);
9460 case MVT::v8f32: return fastEmit_X86ISD_VCVTPS2BF8S_MVT_v8f32_r(RetVT, Op0);
9461 case MVT::v16f32: return fastEmit_X86ISD_VCVTPS2BF8S_MVT_v16f32_r(RetVT, Op0);
9462 default: return Register();
9463 }
9464}
9465
9466// FastEmit functions for X86ISD::VCVTPS2HF8.
9467
9468Register fastEmit_X86ISD_VCVTPS2HF8_MVT_v4f32_r(MVT RetVT, Register Op0) {
9469 if (RetVT.SimpleTy != MVT::v16i8)
9470 return Register();
9471 if ((Subtarget->hasAVX10_V2_AUX())) {
9472 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2HF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9473 }
9474 return Register();
9475}
9476
9477Register fastEmit_X86ISD_VCVTPS2HF8_MVT_v8f32_r(MVT RetVT, Register Op0) {
9478 if (RetVT.SimpleTy != MVT::v16i8)
9479 return Register();
9480 if ((Subtarget->hasAVX10_V2_AUX())) {
9481 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2HF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9482 }
9483 return Register();
9484}
9485
9486Register fastEmit_X86ISD_VCVTPS2HF8_MVT_v16f32_r(MVT RetVT, Register Op0) {
9487 if (RetVT.SimpleTy != MVT::v16i8)
9488 return Register();
9489 if ((Subtarget->hasAVX10_V2_AUX())) {
9490 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2HF8Zrr, RC: &X86::VR128XRegClass, Op0);
9491 }
9492 return Register();
9493}
9494
9495Register fastEmit_X86ISD_VCVTPS2HF8_r(MVT VT, MVT RetVT, Register Op0) {
9496 switch (VT.SimpleTy) {
9497 case MVT::v4f32: return fastEmit_X86ISD_VCVTPS2HF8_MVT_v4f32_r(RetVT, Op0);
9498 case MVT::v8f32: return fastEmit_X86ISD_VCVTPS2HF8_MVT_v8f32_r(RetVT, Op0);
9499 case MVT::v16f32: return fastEmit_X86ISD_VCVTPS2HF8_MVT_v16f32_r(RetVT, Op0);
9500 default: return Register();
9501 }
9502}
9503
9504// FastEmit functions for X86ISD::VCVTPS2HF8S.
9505
9506Register fastEmit_X86ISD_VCVTPS2HF8S_MVT_v4f32_r(MVT RetVT, Register Op0) {
9507 if (RetVT.SimpleTy != MVT::v16i8)
9508 return Register();
9509 if ((Subtarget->hasAVX10_V2_AUX())) {
9510 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9511 }
9512 return Register();
9513}
9514
9515Register fastEmit_X86ISD_VCVTPS2HF8S_MVT_v8f32_r(MVT RetVT, Register Op0) {
9516 if (RetVT.SimpleTy != MVT::v16i8)
9517 return Register();
9518 if ((Subtarget->hasAVX10_V2_AUX())) {
9519 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2HF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9520 }
9521 return Register();
9522}
9523
9524Register fastEmit_X86ISD_VCVTPS2HF8S_MVT_v16f32_r(MVT RetVT, Register Op0) {
9525 if (RetVT.SimpleTy != MVT::v16i8)
9526 return Register();
9527 if ((Subtarget->hasAVX10_V2_AUX())) {
9528 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2HF8SZrr, RC: &X86::VR128XRegClass, Op0);
9529 }
9530 return Register();
9531}
9532
9533Register fastEmit_X86ISD_VCVTPS2HF8S_r(MVT VT, MVT RetVT, Register Op0) {
9534 switch (VT.SimpleTy) {
9535 case MVT::v4f32: return fastEmit_X86ISD_VCVTPS2HF8S_MVT_v4f32_r(RetVT, Op0);
9536 case MVT::v8f32: return fastEmit_X86ISD_VCVTPS2HF8S_MVT_v8f32_r(RetVT, Op0);
9537 case MVT::v16f32: return fastEmit_X86ISD_VCVTPS2HF8S_MVT_v16f32_r(RetVT, Op0);
9538 default: return Register();
9539 }
9540}
9541
9542// FastEmit functions for X86ISD::VCVTROPS2HF8.
9543
9544Register fastEmit_X86ISD_VCVTROPS2HF8_MVT_v4f32_r(MVT RetVT, Register Op0) {
9545 if (RetVT.SimpleTy != MVT::v16i8)
9546 return Register();
9547 if ((Subtarget->hasAVX10_V2_AUX())) {
9548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTROPS2HF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9549 }
9550 return Register();
9551}
9552
9553Register fastEmit_X86ISD_VCVTROPS2HF8_MVT_v8f32_r(MVT RetVT, Register Op0) {
9554 if (RetVT.SimpleTy != MVT::v16i8)
9555 return Register();
9556 if ((Subtarget->hasAVX10_V2_AUX())) {
9557 return fastEmitInst_r(MachineInstOpcode: X86::VCVTROPS2HF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9558 }
9559 return Register();
9560}
9561
9562Register fastEmit_X86ISD_VCVTROPS2HF8_MVT_v16f32_r(MVT RetVT, Register Op0) {
9563 if (RetVT.SimpleTy != MVT::v16i8)
9564 return Register();
9565 if ((Subtarget->hasAVX10_V2_AUX())) {
9566 return fastEmitInst_r(MachineInstOpcode: X86::VCVTROPS2HF8Zrr, RC: &X86::VR128XRegClass, Op0);
9567 }
9568 return Register();
9569}
9570
9571Register fastEmit_X86ISD_VCVTROPS2HF8_r(MVT VT, MVT RetVT, Register Op0) {
9572 switch (VT.SimpleTy) {
9573 case MVT::v4f32: return fastEmit_X86ISD_VCVTROPS2HF8_MVT_v4f32_r(RetVT, Op0);
9574 case MVT::v8f32: return fastEmit_X86ISD_VCVTROPS2HF8_MVT_v8f32_r(RetVT, Op0);
9575 case MVT::v16f32: return fastEmit_X86ISD_VCVTROPS2HF8_MVT_v16f32_r(RetVT, Op0);
9576 default: return Register();
9577 }
9578}
9579
9580// FastEmit functions for X86ISD::VCVTROPS2HF8S.
9581
9582Register fastEmit_X86ISD_VCVTROPS2HF8S_MVT_v4f32_r(MVT RetVT, Register Op0) {
9583 if (RetVT.SimpleTy != MVT::v16i8)
9584 return Register();
9585 if ((Subtarget->hasAVX10_V2_AUX())) {
9586 return fastEmitInst_r(MachineInstOpcode: X86::VCVTROPS2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9587 }
9588 return Register();
9589}
9590
9591Register fastEmit_X86ISD_VCVTROPS2HF8S_MVT_v8f32_r(MVT RetVT, Register Op0) {
9592 if (RetVT.SimpleTy != MVT::v16i8)
9593 return Register();
9594 if ((Subtarget->hasAVX10_V2_AUX())) {
9595 return fastEmitInst_r(MachineInstOpcode: X86::VCVTROPS2HF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9596 }
9597 return Register();
9598}
9599
9600Register fastEmit_X86ISD_VCVTROPS2HF8S_MVT_v16f32_r(MVT RetVT, Register Op0) {
9601 if (RetVT.SimpleTy != MVT::v16i8)
9602 return Register();
9603 if ((Subtarget->hasAVX10_V2_AUX())) {
9604 return fastEmitInst_r(MachineInstOpcode: X86::VCVTROPS2HF8SZrr, RC: &X86::VR128XRegClass, Op0);
9605 }
9606 return Register();
9607}
9608
9609Register fastEmit_X86ISD_VCVTROPS2HF8S_r(MVT VT, MVT RetVT, Register Op0) {
9610 switch (VT.SimpleTy) {
9611 case MVT::v4f32: return fastEmit_X86ISD_VCVTROPS2HF8S_MVT_v4f32_r(RetVT, Op0);
9612 case MVT::v8f32: return fastEmit_X86ISD_VCVTROPS2HF8S_MVT_v8f32_r(RetVT, Op0);
9613 case MVT::v16f32: return fastEmit_X86ISD_VCVTROPS2HF8S_MVT_v16f32_r(RetVT, Op0);
9614 default: return Register();
9615 }
9616}
9617
9618// FastEmit functions for X86ISD::VFPEXT.
9619
9620Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Register Op0) {
9621 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9622 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
9623 }
9624 return Register();
9625}
9626
9627Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Register Op0) {
9628 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9629 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
9630 }
9631 return Register();
9632}
9633
9634Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Register Op0) {
9635 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9636 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
9637 }
9638 return Register();
9639}
9640
9641Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(MVT RetVT, Register Op0) {
9642switch (RetVT.SimpleTy) {
9643 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
9644 case MVT::v2f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
9645 case MVT::v4f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
9646 default: return Register();
9647}
9648}
9649
9650Register fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, Register Op0) {
9651 if (RetVT.SimpleTy != MVT::v2f64)
9652 return Register();
9653 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9654 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
9655 }
9656 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9657 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
9658 }
9659 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9660 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
9661 }
9662 return Register();
9663}
9664
9665Register fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, Register Op0) {
9666 switch (VT.SimpleTy) {
9667 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(RetVT, Op0);
9668 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0);
9669 default: return Register();
9670 }
9671}
9672
9673// FastEmit functions for X86ISD::VFPEXT_SAE.
9674
9675Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
9676 if (RetVT.SimpleTy != MVT::v8f64)
9677 return Register();
9678 if ((Subtarget->hasFP16())) {
9679 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrrb, RC: &X86::VR512RegClass, Op0);
9680 }
9681 return Register();
9682}
9683
9684Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
9685 if (RetVT.SimpleTy != MVT::v16f32)
9686 return Register();
9687 if ((Subtarget->hasFP16())) {
9688 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrrb, RC: &X86::VR512RegClass, Op0);
9689 }
9690 return Register();
9691}
9692
9693Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
9694 if (RetVT.SimpleTy != MVT::v8f64)
9695 return Register();
9696 if ((Subtarget->hasAVX512())) {
9697 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrrb, RC: &X86::VR512RegClass, Op0);
9698 }
9699 return Register();
9700}
9701
9702Register fastEmit_X86ISD_VFPEXT_SAE_r(MVT VT, MVT RetVT, Register Op0) {
9703 switch (VT.SimpleTy) {
9704 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(RetVT, Op0);
9705 case MVT::v16f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(RetVT, Op0);
9706 case MVT::v8f32: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(RetVT, Op0);
9707 default: return Register();
9708 }
9709}
9710
9711// FastEmit functions for X86ISD::VFPROUND.
9712
9713Register fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
9714 if (RetVT.SimpleTy != MVT::v8f16)
9715 return Register();
9716 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9717 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
9718 }
9719 return Register();
9720}
9721
9722Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Register Op0) {
9723 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9724 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
9725 }
9726 return Register();
9727}
9728
9729Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Register Op0) {
9730 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9731 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
9732 }
9733 if ((Subtarget->hasAVXNECONVERT())) {
9734 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Yrr, RC: &X86::VR128RegClass, Op0);
9735 }
9736 return Register();
9737}
9738
9739Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(MVT RetVT, Register Op0) {
9740switch (RetVT.SimpleTy) {
9741 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Op0);
9742 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Op0);
9743 default: return Register();
9744}
9745}
9746
9747Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Register Op0) {
9748 if ((Subtarget->hasFP16())) {
9749 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
9750 }
9751 return Register();
9752}
9753
9754Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Register Op0) {
9755 if ((Subtarget->hasBF16())) {
9756 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
9757 }
9758 return Register();
9759}
9760
9761Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(MVT RetVT, Register Op0) {
9762switch (RetVT.SimpleTy) {
9763 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Op0);
9764 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Op0);
9765 default: return Register();
9766}
9767}
9768
9769Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Register Op0) {
9770 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9771 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
9772 }
9773 return Register();
9774}
9775
9776Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Register Op0) {
9777 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9778 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
9779 }
9780 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9781 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
9782 }
9783 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9784 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
9785 }
9786 return Register();
9787}
9788
9789Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
9790switch (RetVT.SimpleTy) {
9791 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
9792 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
9793 default: return Register();
9794}
9795}
9796
9797Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Register Op0) {
9798 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9799 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
9800 }
9801 return Register();
9802}
9803
9804Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Register Op0) {
9805 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9806 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
9807 }
9808 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9809 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
9810 }
9811 return Register();
9812}
9813
9814Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(MVT RetVT, Register Op0) {
9815switch (RetVT.SimpleTy) {
9816 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
9817 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
9818 default: return Register();
9819}
9820}
9821
9822Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Register Op0) {
9823 if ((Subtarget->hasFP16())) {
9824 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
9825 }
9826 return Register();
9827}
9828
9829Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Register Op0) {
9830 if ((Subtarget->hasAVX512())) {
9831 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
9832 }
9833 return Register();
9834}
9835
9836Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(MVT RetVT, Register Op0) {
9837switch (RetVT.SimpleTy) {
9838 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
9839 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
9840 default: return Register();
9841}
9842}
9843
9844Register fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, Register Op0) {
9845 switch (VT.SimpleTy) {
9846 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(RetVT, Op0);
9847 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(RetVT, Op0);
9848 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(RetVT, Op0);
9849 case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0);
9850 case MVT::v4f64: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(RetVT, Op0);
9851 case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(RetVT, Op0);
9852 default: return Register();
9853 }
9854}
9855
9856// FastEmit functions for X86ISD::VTRUNC.
9857
9858Register fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, Register Op0) {
9859 if (RetVT.SimpleTy != MVT::v16i8)
9860 return Register();
9861 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9862 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ128rr, RC: &X86::VR128XRegClass, Op0);
9863 }
9864 return Register();
9865}
9866
9867Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Register Op0) {
9868 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9869 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ128rr, RC: &X86::VR128XRegClass, Op0);
9870 }
9871 return Register();
9872}
9873
9874Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Register Op0) {
9875 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9876 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ128rr, RC: &X86::VR128XRegClass, Op0);
9877 }
9878 return Register();
9879}
9880
9881Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, Register Op0) {
9882switch (RetVT.SimpleTy) {
9883 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0);
9884 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0);
9885 default: return Register();
9886}
9887}
9888
9889Register fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, Register Op0) {
9890 if (RetVT.SimpleTy != MVT::v16i8)
9891 return Register();
9892 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9893 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ256rr, RC: &X86::VR128XRegClass, Op0);
9894 }
9895 return Register();
9896}
9897
9898Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Register Op0) {
9899 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9900 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ128rr, RC: &X86::VR128XRegClass, Op0);
9901 }
9902 return Register();
9903}
9904
9905Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Register Op0) {
9906 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9907 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ128rr, RC: &X86::VR128XRegClass, Op0);
9908 }
9909 return Register();
9910}
9911
9912Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Register Op0) {
9913 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9914 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ128rr, RC: &X86::VR128XRegClass, Op0);
9915 }
9916 return Register();
9917}
9918
9919Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, Register Op0) {
9920switch (RetVT.SimpleTy) {
9921 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0);
9922 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0);
9923 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0);
9924 default: return Register();
9925}
9926}
9927
9928Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Register Op0) {
9929 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9930 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ256rr, RC: &X86::VR128XRegClass, Op0);
9931 }
9932 return Register();
9933}
9934
9935Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Register Op0) {
9936 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9937 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ256rr, RC: &X86::VR128XRegClass, Op0);
9938 }
9939 return Register();
9940}
9941
9942Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, Register Op0) {
9943switch (RetVT.SimpleTy) {
9944 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0);
9945 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0);
9946 default: return Register();
9947}
9948}
9949
9950Register fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, Register Op0) {
9951 if (RetVT.SimpleTy != MVT::v16i8)
9952 return Register();
9953 if ((Subtarget->hasAVX512())) {
9954 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZrr, RC: &X86::VR128XRegClass, Op0);
9955 }
9956 return Register();
9957}
9958
9959Register fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
9960 switch (VT.SimpleTy) {
9961 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0);
9962 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0);
9963 case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0);
9964 case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0);
9965 case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0);
9966 case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0);
9967 default: return Register();
9968 }
9969}
9970
9971// FastEmit functions for X86ISD::VTRUNCS.
9972
9973Register fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, Register Op0) {
9974 if (RetVT.SimpleTy != MVT::v16i8)
9975 return Register();
9976 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9977 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
9978 }
9979 return Register();
9980}
9981
9982Register fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, Register Op0) {
9983 if (RetVT.SimpleTy != MVT::v16i8)
9984 return Register();
9985 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9986 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
9987 }
9988 return Register();
9989}
9990
9991Register fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, Register Op0) {
9992 if (RetVT.SimpleTy != MVT::v32i8)
9993 return Register();
9994 if ((Subtarget->hasBWI())) {
9995 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZrr, RC: &X86::VR256XRegClass, Op0);
9996 }
9997 return Register();
9998}
9999
10000Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Register Op0) {
10001 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10002 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
10003 }
10004 return Register();
10005}
10006
10007Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Register Op0) {
10008 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10009 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
10010 }
10011 return Register();
10012}
10013
10014Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, Register Op0) {
10015switch (RetVT.SimpleTy) {
10016 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0);
10017 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0);
10018 default: return Register();
10019}
10020}
10021
10022Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Register Op0) {
10023 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10024 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
10025 }
10026 return Register();
10027}
10028
10029Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Register Op0) {
10030 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10031 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
10032 }
10033 return Register();
10034}
10035
10036Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, Register Op0) {
10037switch (RetVT.SimpleTy) {
10038 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0);
10039 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0);
10040 default: return Register();
10041}
10042}
10043
10044Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Register Op0) {
10045 if ((Subtarget->hasAVX512())) {
10046 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZrr, RC: &X86::VR128XRegClass, Op0);
10047 }
10048 return Register();
10049}
10050
10051Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Register Op0) {
10052 if ((Subtarget->hasAVX512())) {
10053 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZrr, RC: &X86::VR256XRegClass, Op0);
10054 }
10055 return Register();
10056}
10057
10058Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, Register Op0) {
10059switch (RetVT.SimpleTy) {
10060 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0);
10061 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0);
10062 default: return Register();
10063}
10064}
10065
10066Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Register Op0) {
10067 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10068 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
10069 }
10070 return Register();
10071}
10072
10073Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Register Op0) {
10074 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10075 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
10076 }
10077 return Register();
10078}
10079
10080Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Register Op0) {
10081 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10082 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
10083 }
10084 return Register();
10085}
10086
10087Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, Register Op0) {
10088switch (RetVT.SimpleTy) {
10089 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0);
10090 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0);
10091 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0);
10092 default: return Register();
10093}
10094}
10095
10096Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Register Op0) {
10097 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10098 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
10099 }
10100 return Register();
10101}
10102
10103Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Register Op0) {
10104 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10105 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
10106 }
10107 return Register();
10108}
10109
10110Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Register Op0) {
10111 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10112 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
10113 }
10114 return Register();
10115}
10116
10117Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, Register Op0) {
10118switch (RetVT.SimpleTy) {
10119 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0);
10120 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0);
10121 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0);
10122 default: return Register();
10123}
10124}
10125
10126Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Register Op0) {
10127 if ((Subtarget->hasAVX512())) {
10128 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZrr, RC: &X86::VR128XRegClass, Op0);
10129 }
10130 return Register();
10131}
10132
10133Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Register Op0) {
10134 if ((Subtarget->hasAVX512())) {
10135 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZrr, RC: &X86::VR128XRegClass, Op0);
10136 }
10137 return Register();
10138}
10139
10140Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Register Op0) {
10141 if ((Subtarget->hasAVX512())) {
10142 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZrr, RC: &X86::VR256XRegClass, Op0);
10143 }
10144 return Register();
10145}
10146
10147Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, Register Op0) {
10148switch (RetVT.SimpleTy) {
10149 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0);
10150 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0);
10151 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0);
10152 default: return Register();
10153}
10154}
10155
10156Register fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, Register Op0) {
10157 switch (VT.SimpleTy) {
10158 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0);
10159 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0);
10160 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0);
10161 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0);
10162 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0);
10163 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0);
10164 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0);
10165 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0);
10166 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0);
10167 default: return Register();
10168 }
10169}
10170
10171// FastEmit functions for X86ISD::VTRUNCSS.
10172
10173Register fastEmit_X86ISD_VTRUNCSS_MVT_v4i32_r(MVT RetVT, Register Op0) {
10174 if (RetVT.SimpleTy != MVT::v16i8)
10175 return Register();
10176 if ((Subtarget->hasAVX10_V2_AUX())) {
10177 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
10178 }
10179 return Register();
10180}
10181
10182Register fastEmit_X86ISD_VTRUNCSS_MVT_v8i32_r(MVT RetVT, Register Op0) {
10183 if (RetVT.SimpleTy != MVT::v16i8)
10184 return Register();
10185 if ((Subtarget->hasAVX10_V2_AUX())) {
10186 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
10187 }
10188 return Register();
10189}
10190
10191Register fastEmit_X86ISD_VTRUNCSS_MVT_v16i32_r(MVT RetVT, Register Op0) {
10192 if (RetVT.SimpleTy != MVT::v16i8)
10193 return Register();
10194 if ((Subtarget->hasAVX10_V2_AUX())) {
10195 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSSDBZrr, RC: &X86::VR128XRegClass, Op0);
10196 }
10197 return Register();
10198}
10199
10200Register fastEmit_X86ISD_VTRUNCSS_r(MVT VT, MVT RetVT, Register Op0) {
10201 switch (VT.SimpleTy) {
10202 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCSS_MVT_v4i32_r(RetVT, Op0);
10203 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCSS_MVT_v8i32_r(RetVT, Op0);
10204 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCSS_MVT_v16i32_r(RetVT, Op0);
10205 default: return Register();
10206 }
10207}
10208
10209// FastEmit functions for X86ISD::VTRUNCUS.
10210
10211Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, Register Op0) {
10212 if (RetVT.SimpleTy != MVT::v16i8)
10213 return Register();
10214 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10215 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
10216 }
10217 return Register();
10218}
10219
10220Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, Register Op0) {
10221 if (RetVT.SimpleTy != MVT::v16i8)
10222 return Register();
10223 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10224 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
10225 }
10226 return Register();
10227}
10228
10229Register fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, Register Op0) {
10230 if (RetVT.SimpleTy != MVT::v32i8)
10231 return Register();
10232 if ((Subtarget->hasBWI())) {
10233 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZrr, RC: &X86::VR256XRegClass, Op0);
10234 }
10235 return Register();
10236}
10237
10238Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Register Op0) {
10239 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10240 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
10241 }
10242 return Register();
10243}
10244
10245Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Register Op0) {
10246 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10247 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
10248 }
10249 return Register();
10250}
10251
10252Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, Register Op0) {
10253switch (RetVT.SimpleTy) {
10254 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0);
10255 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0);
10256 default: return Register();
10257}
10258}
10259
10260Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Register Op0) {
10261 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10262 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
10263 }
10264 return Register();
10265}
10266
10267Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Register Op0) {
10268 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10269 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
10270 }
10271 return Register();
10272}
10273
10274Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, Register Op0) {
10275switch (RetVT.SimpleTy) {
10276 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0);
10277 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0);
10278 default: return Register();
10279}
10280}
10281
10282Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Register Op0) {
10283 if ((Subtarget->hasAVX512())) {
10284 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZrr, RC: &X86::VR128XRegClass, Op0);
10285 }
10286 return Register();
10287}
10288
10289Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Register Op0) {
10290 if ((Subtarget->hasAVX512())) {
10291 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZrr, RC: &X86::VR256XRegClass, Op0);
10292 }
10293 return Register();
10294}
10295
10296Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, Register Op0) {
10297switch (RetVT.SimpleTy) {
10298 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0);
10299 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0);
10300 default: return Register();
10301}
10302}
10303
10304Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Register Op0) {
10305 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10306 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
10307 }
10308 return Register();
10309}
10310
10311Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Register Op0) {
10312 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10313 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
10314 }
10315 return Register();
10316}
10317
10318Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Register Op0) {
10319 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10320 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
10321 }
10322 return Register();
10323}
10324
10325Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, Register Op0) {
10326switch (RetVT.SimpleTy) {
10327 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0);
10328 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0);
10329 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0);
10330 default: return Register();
10331}
10332}
10333
10334Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Register Op0) {
10335 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10336 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
10337 }
10338 return Register();
10339}
10340
10341Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Register Op0) {
10342 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10343 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
10344 }
10345 return Register();
10346}
10347
10348Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Register Op0) {
10349 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10350 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
10351 }
10352 return Register();
10353}
10354
10355Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, Register Op0) {
10356switch (RetVT.SimpleTy) {
10357 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0);
10358 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0);
10359 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0);
10360 default: return Register();
10361}
10362}
10363
10364Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Register Op0) {
10365 if ((Subtarget->hasAVX512())) {
10366 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZrr, RC: &X86::VR128XRegClass, Op0);
10367 }
10368 return Register();
10369}
10370
10371Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Register Op0) {
10372 if ((Subtarget->hasAVX512())) {
10373 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZrr, RC: &X86::VR128XRegClass, Op0);
10374 }
10375 return Register();
10376}
10377
10378Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Register Op0) {
10379 if ((Subtarget->hasAVX512())) {
10380 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZrr, RC: &X86::VR256XRegClass, Op0);
10381 }
10382 return Register();
10383}
10384
10385Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, Register Op0) {
10386switch (RetVT.SimpleTy) {
10387 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0);
10388 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0);
10389 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0);
10390 default: return Register();
10391}
10392}
10393
10394Register fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, Register Op0) {
10395 switch (VT.SimpleTy) {
10396 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0);
10397 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0);
10398 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0);
10399 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0);
10400 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0);
10401 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0);
10402 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0);
10403 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0);
10404 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0);
10405 default: return Register();
10406 }
10407}
10408
10409// FastEmit functions for X86ISD::VZEXT_MOVL.
10410
10411Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v8i16_r(MVT RetVT, Register Op0) {
10412 if (RetVT.SimpleTy != MVT::v8i16)
10413 return Register();
10414 if ((Subtarget->hasAVX10_2())) {
10415 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPWILo2PWIZrr, RC: &X86::VR128XRegClass, Op0);
10416 }
10417 return Register();
10418}
10419
10420Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v4i32_r(MVT RetVT, Register Op0) {
10421 if (RetVT.SimpleTy != MVT::v4i32)
10422 return Register();
10423 if ((Subtarget->hasAVX10_2())) {
10424 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPDILo2PDIZrr, RC: &X86::VR128XRegClass, Op0);
10425 }
10426 return Register();
10427}
10428
10429Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, Register Op0) {
10430 if (RetVT.SimpleTy != MVT::v2i64)
10431 return Register();
10432 if ((Subtarget->hasAVX512())) {
10433 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
10434 }
10435 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10436 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10437 }
10438 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10439 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10440 }
10441 return Register();
10442}
10443
10444Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, Register Op0) {
10445 if (RetVT.SimpleTy != MVT::v2f64)
10446 return Register();
10447 if ((Subtarget->hasAVX512())) {
10448 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
10449 }
10450 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10451 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10452 }
10453 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10454 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10455 }
10456 return Register();
10457}
10458
10459Register fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, Register Op0) {
10460 switch (VT.SimpleTy) {
10461 case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v8i16_r(RetVT, Op0);
10462 case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v4i32_r(RetVT, Op0);
10463 case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0);
10464 case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0);
10465 default: return Register();
10466 }
10467}
10468
10469// Top-level FastEmit function.
10470
10471Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
10472 switch (Opcode) {
10473 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
10474 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
10475 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
10476 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
10477 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
10478 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
10479 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
10480 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
10481 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
10482 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
10483 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
10484 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
10485 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
10486 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
10487 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
10488 case ISD::LLRINT: return fastEmit_ISD_LLRINT_r(VT, RetVT, Op0);
10489 case ISD::LRINT: return fastEmit_ISD_LRINT_r(VT, RetVT, Op0);
10490 case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0);
10491 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
10492 case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
10493 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
10494 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
10495 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
10496 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
10497 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
10498 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
10499 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
10500 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
10501 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
10502 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
10503 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
10504 case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
10505 case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0);
10506 case X86ISD::CVTNEPS2BF16: return fastEmit_X86ISD_CVTNEPS2BF16_r(VT, RetVT, Op0);
10507 case X86ISD::CVTP2IBS: return fastEmit_X86ISD_CVTP2IBS_r(VT, RetVT, Op0);
10508 case X86ISD::CVTP2IUBS: return fastEmit_X86ISD_CVTP2IUBS_r(VT, RetVT, Op0);
10509 case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0);
10510 case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0);
10511 case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0);
10512 case X86ISD::CVTPH2PS_SAE: return fastEmit_X86ISD_CVTPH2PS_SAE_r(VT, RetVT, Op0);
10513 case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0);
10514 case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0);
10515 case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0);
10516 case X86ISD::CVTTP2IBS: return fastEmit_X86ISD_CVTTP2IBS_r(VT, RetVT, Op0);
10517 case X86ISD::CVTTP2IBS_SAE: return fastEmit_X86ISD_CVTTP2IBS_SAE_r(VT, RetVT, Op0);
10518 case X86ISD::CVTTP2IUBS: return fastEmit_X86ISD_CVTTP2IUBS_r(VT, RetVT, Op0);
10519 case X86ISD::CVTTP2IUBS_SAE: return fastEmit_X86ISD_CVTTP2IUBS_SAE_r(VT, RetVT, Op0);
10520 case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0);
10521 case X86ISD::CVTTP2SIS: return fastEmit_X86ISD_CVTTP2SIS_r(VT, RetVT, Op0);
10522 case X86ISD::CVTTP2SIS_SAE: return fastEmit_X86ISD_CVTTP2SIS_SAE_r(VT, RetVT, Op0);
10523 case X86ISD::CVTTP2SI_SAE: return fastEmit_X86ISD_CVTTP2SI_SAE_r(VT, RetVT, Op0);
10524 case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0);
10525 case X86ISD::CVTTP2UIS: return fastEmit_X86ISD_CVTTP2UIS_r(VT, RetVT, Op0);
10526 case X86ISD::CVTTP2UIS_SAE: return fastEmit_X86ISD_CVTTP2UIS_SAE_r(VT, RetVT, Op0);
10527 case X86ISD::CVTTP2UI_SAE: return fastEmit_X86ISD_CVTTP2UI_SAE_r(VT, RetVT, Op0);
10528 case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0);
10529 case X86ISD::CVTTS2SIS: return fastEmit_X86ISD_CVTTS2SIS_r(VT, RetVT, Op0);
10530 case X86ISD::CVTTS2SIS_SAE: return fastEmit_X86ISD_CVTTS2SIS_SAE_r(VT, RetVT, Op0);
10531 case X86ISD::CVTTS2SI_SAE: return fastEmit_X86ISD_CVTTS2SI_SAE_r(VT, RetVT, Op0);
10532 case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0);
10533 case X86ISD::CVTTS2UIS: return fastEmit_X86ISD_CVTTS2UIS_r(VT, RetVT, Op0);
10534 case X86ISD::CVTTS2UIS_SAE: return fastEmit_X86ISD_CVTTS2UIS_SAE_r(VT, RetVT, Op0);
10535 case X86ISD::CVTTS2UI_SAE: return fastEmit_X86ISD_CVTTS2UI_SAE_r(VT, RetVT, Op0);
10536 case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0);
10537 case X86ISD::FGETEXP: return fastEmit_X86ISD_FGETEXP_r(VT, RetVT, Op0);
10538 case X86ISD::FGETEXP_SAE: return fastEmit_X86ISD_FGETEXP_SAE_r(VT, RetVT, Op0);
10539 case X86ISD::FP_TO_SINT_SAT: return fastEmit_X86ISD_FP_TO_SINT_SAT_r(VT, RetVT, Op0);
10540 case X86ISD::FP_TO_UINT_SAT: return fastEmit_X86ISD_FP_TO_UINT_SAT_r(VT, RetVT, Op0);
10541 case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0);
10542 case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0);
10543 case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0);
10544 case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0);
10545 case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0);
10546 case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0);
10547 case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0);
10548 case X86ISD::MOVQ2DQ: return fastEmit_X86ISD_MOVQ2DQ_r(VT, RetVT, Op0);
10549 case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0);
10550 case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0);
10551 case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0);
10552 case X86ISD::PROBED_ALLOCA: return fastEmit_X86ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
10553 case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0);
10554 case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0);
10555 case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0);
10556 case X86ISD::STRICT_CVTPH2PS: return fastEmit_X86ISD_STRICT_CVTPH2PS_r(VT, RetVT, Op0);
10557 case X86ISD::STRICT_CVTSI2P: return fastEmit_X86ISD_STRICT_CVTSI2P_r(VT, RetVT, Op0);
10558 case X86ISD::STRICT_CVTTP2SI: return fastEmit_X86ISD_STRICT_CVTTP2SI_r(VT, RetVT, Op0);
10559 case X86ISD::STRICT_CVTTP2UI: return fastEmit_X86ISD_STRICT_CVTTP2UI_r(VT, RetVT, Op0);
10560 case X86ISD::STRICT_CVTUI2P: return fastEmit_X86ISD_STRICT_CVTUI2P_r(VT, RetVT, Op0);
10561 case X86ISD::STRICT_VFPEXT: return fastEmit_X86ISD_STRICT_VFPEXT_r(VT, RetVT, Op0);
10562 case X86ISD::STRICT_VFPROUND: return fastEmit_X86ISD_STRICT_VFPROUND_r(VT, RetVT, Op0);
10563 case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0);
10564 case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0);
10565 case X86ISD::VCVTBF82PS: return fastEmit_X86ISD_VCVTBF82PS_r(VT, RetVT, Op0);
10566 case X86ISD::VCVTHF82PH: return fastEmit_X86ISD_VCVTHF82PH_r(VT, RetVT, Op0);
10567 case X86ISD::VCVTHF82PS: return fastEmit_X86ISD_VCVTHF82PS_r(VT, RetVT, Op0);
10568 case X86ISD::VCVTPH2BF8: return fastEmit_X86ISD_VCVTPH2BF8_r(VT, RetVT, Op0);
10569 case X86ISD::VCVTPH2BF8S: return fastEmit_X86ISD_VCVTPH2BF8S_r(VT, RetVT, Op0);
10570 case X86ISD::VCVTPH2HF8: return fastEmit_X86ISD_VCVTPH2HF8_r(VT, RetVT, Op0);
10571 case X86ISD::VCVTPH2HF8S: return fastEmit_X86ISD_VCVTPH2HF8S_r(VT, RetVT, Op0);
10572 case X86ISD::VCVTPS2BF8: return fastEmit_X86ISD_VCVTPS2BF8_r(VT, RetVT, Op0);
10573 case X86ISD::VCVTPS2BF8S: return fastEmit_X86ISD_VCVTPS2BF8S_r(VT, RetVT, Op0);
10574 case X86ISD::VCVTPS2HF8: return fastEmit_X86ISD_VCVTPS2HF8_r(VT, RetVT, Op0);
10575 case X86ISD::VCVTPS2HF8S: return fastEmit_X86ISD_VCVTPS2HF8S_r(VT, RetVT, Op0);
10576 case X86ISD::VCVTROPS2HF8: return fastEmit_X86ISD_VCVTROPS2HF8_r(VT, RetVT, Op0);
10577 case X86ISD::VCVTROPS2HF8S: return fastEmit_X86ISD_VCVTROPS2HF8S_r(VT, RetVT, Op0);
10578 case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0);
10579 case X86ISD::VFPEXT_SAE: return fastEmit_X86ISD_VFPEXT_SAE_r(VT, RetVT, Op0);
10580 case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0);
10581 case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0);
10582 case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0);
10583 case X86ISD::VTRUNCSS: return fastEmit_X86ISD_VTRUNCSS_r(VT, RetVT, Op0);
10584 case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0);
10585 case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0);
10586 default: return Register();
10587 }
10588}
10589
10590// FastEmit functions for ISD::ADD.
10591
10592Register fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
10593 if (RetVT.SimpleTy != MVT::i8)
10594 return Register();
10595 if ((Subtarget->hasNDD())) {
10596 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10597 }
10598 if ((!Subtarget->hasNDD())) {
10599 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr, RC: &X86::GR8RegClass, Op0, Op1);
10600 }
10601 return Register();
10602}
10603
10604Register fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
10605 if (RetVT.SimpleTy != MVT::i16)
10606 return Register();
10607 if ((Subtarget->hasNDD())) {
10608 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10609 }
10610 if ((!Subtarget->hasNDD())) {
10611 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr, RC: &X86::GR16RegClass, Op0, Op1);
10612 }
10613 return Register();
10614}
10615
10616Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10617 if (RetVT.SimpleTy != MVT::i32)
10618 return Register();
10619 if ((Subtarget->hasNDD())) {
10620 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10621 }
10622 if ((!Subtarget->hasNDD())) {
10623 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr, RC: &X86::GR32RegClass, Op0, Op1);
10624 }
10625 return Register();
10626}
10627
10628Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10629 if (RetVT.SimpleTy != MVT::i64)
10630 return Register();
10631 if ((Subtarget->hasNDD())) {
10632 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10633 }
10634 if ((!Subtarget->hasNDD())) {
10635 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr, RC: &X86::GR64RegClass, Op0, Op1);
10636 }
10637 return Register();
10638}
10639
10640Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10641 if (RetVT.SimpleTy != MVT::v16i8)
10642 return Register();
10643 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10644 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10645 }
10646 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10647 return fastEmitInst_rr(MachineInstOpcode: X86::PADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
10648 }
10649 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10650 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
10651 }
10652 return Register();
10653}
10654
10655Register fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10656 if (RetVT.SimpleTy != MVT::v32i8)
10657 return Register();
10658 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10659 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10660 }
10661 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10662 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10663 }
10664 return Register();
10665}
10666
10667Register fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10668 if (RetVT.SimpleTy != MVT::v64i8)
10669 return Register();
10670 if ((Subtarget->hasBWI())) {
10671 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10672 }
10673 return Register();
10674}
10675
10676Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10677 if (RetVT.SimpleTy != MVT::v8i16)
10678 return Register();
10679 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10680 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10681 }
10682 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10683 return fastEmitInst_rr(MachineInstOpcode: X86::PADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
10684 }
10685 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10686 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
10687 }
10688 return Register();
10689}
10690
10691Register fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10692 if (RetVT.SimpleTy != MVT::v16i16)
10693 return Register();
10694 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10695 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10696 }
10697 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10698 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10699 }
10700 return Register();
10701}
10702
10703Register fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10704 if (RetVT.SimpleTy != MVT::v32i16)
10705 return Register();
10706 if ((Subtarget->hasBWI())) {
10707 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10708 }
10709 return Register();
10710}
10711
10712Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10713 if (RetVT.SimpleTy != MVT::v4i32)
10714 return Register();
10715 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10716 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10717 }
10718 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10719 return fastEmitInst_rr(MachineInstOpcode: X86::PADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
10720 }
10721 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10722 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
10723 }
10724 return Register();
10725}
10726
10727Register fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
10728 if (RetVT.SimpleTy != MVT::v8i32)
10729 return Register();
10730 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10731 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10732 }
10733 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10734 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10735 }
10736 return Register();
10737}
10738
10739Register fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
10740 if (RetVT.SimpleTy != MVT::v16i32)
10741 return Register();
10742 if ((Subtarget->hasAVX512())) {
10743 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10744 }
10745 return Register();
10746}
10747
10748Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10749 if (RetVT.SimpleTy != MVT::v2i64)
10750 return Register();
10751 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10752 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10753 }
10754 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10755 return fastEmitInst_rr(MachineInstOpcode: X86::PADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
10756 }
10757 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10758 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
10759 }
10760 return Register();
10761}
10762
10763Register fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
10764 if (RetVT.SimpleTy != MVT::v4i64)
10765 return Register();
10766 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10767 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10768 }
10769 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10770 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
10771 }
10772 return Register();
10773}
10774
10775Register fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
10776 if (RetVT.SimpleTy != MVT::v8i64)
10777 return Register();
10778 if ((Subtarget->hasAVX512())) {
10779 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10780 }
10781 return Register();
10782}
10783
10784Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10785 switch (VT.SimpleTy) {
10786 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op1);
10787 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op1);
10788 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10789 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10790 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10791 case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op1);
10792 case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op1);
10793 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10794 case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op1);
10795 case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op1);
10796 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10797 case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op1);
10798 case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op1);
10799 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10800 case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op1);
10801 case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op1);
10802 default: return Register();
10803 }
10804}
10805
10806// FastEmit functions for ISD::AND.
10807
10808Register fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
10809 if (RetVT.SimpleTy != MVT::i8)
10810 return Register();
10811 if ((Subtarget->hasNDD())) {
10812 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10813 }
10814 if ((!Subtarget->hasNDD())) {
10815 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr, RC: &X86::GR8RegClass, Op0, Op1);
10816 }
10817 return Register();
10818}
10819
10820Register fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
10821 if (RetVT.SimpleTy != MVT::i16)
10822 return Register();
10823 if ((Subtarget->hasNDD())) {
10824 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10825 }
10826 if ((!Subtarget->hasNDD())) {
10827 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr, RC: &X86::GR16RegClass, Op0, Op1);
10828 }
10829 return Register();
10830}
10831
10832Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10833 if (RetVT.SimpleTy != MVT::i32)
10834 return Register();
10835 if ((Subtarget->hasNDD())) {
10836 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10837 }
10838 if ((!Subtarget->hasNDD())) {
10839 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr, RC: &X86::GR32RegClass, Op0, Op1);
10840 }
10841 return Register();
10842}
10843
10844Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10845 if (RetVT.SimpleTy != MVT::i64)
10846 return Register();
10847 if ((Subtarget->hasNDD())) {
10848 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10849 }
10850 if ((!Subtarget->hasNDD())) {
10851 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr, RC: &X86::GR64RegClass, Op0, Op1);
10852 }
10853 return Register();
10854}
10855
10856Register fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10857 if (RetVT.SimpleTy != MVT::v8i1)
10858 return Register();
10859 if ((Subtarget->hasDQI())) {
10860 return fastEmitInst_rr(MachineInstOpcode: X86::KANDBkk, RC: &X86::VK8RegClass, Op0, Op1);
10861 }
10862 return Register();
10863}
10864
10865Register fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
10866 if (RetVT.SimpleTy != MVT::v16i1)
10867 return Register();
10868 if ((Subtarget->hasAVX512())) {
10869 return fastEmitInst_rr(MachineInstOpcode: X86::KANDWkk, RC: &X86::VK16RegClass, Op0, Op1);
10870 }
10871 return Register();
10872}
10873
10874Register fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
10875 if (RetVT.SimpleTy != MVT::v32i1)
10876 return Register();
10877 if ((Subtarget->hasBWI())) {
10878 return fastEmitInst_rr(MachineInstOpcode: X86::KANDDkk, RC: &X86::VK32RegClass, Op0, Op1);
10879 }
10880 return Register();
10881}
10882
10883Register fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
10884 if (RetVT.SimpleTy != MVT::v64i1)
10885 return Register();
10886 if ((Subtarget->hasBWI())) {
10887 return fastEmitInst_rr(MachineInstOpcode: X86::KANDQkk, RC: &X86::VK64RegClass, Op0, Op1);
10888 }
10889 return Register();
10890}
10891
10892Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10893 if (RetVT.SimpleTy != MVT::v16i8)
10894 return Register();
10895 if ((Subtarget->hasVLX())) {
10896 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10897 }
10898 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10899 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10900 }
10901 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10902 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10903 }
10904 return Register();
10905}
10906
10907Register fastEmit_ISD_AND_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10908 if (RetVT.SimpleTy != MVT::v32i8)
10909 return Register();
10910 if ((Subtarget->hasVLX())) {
10911 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10912 }
10913 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10914 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10915 }
10916 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10917 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10918 }
10919 return Register();
10920}
10921
10922Register fastEmit_ISD_AND_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10923 if (RetVT.SimpleTy != MVT::v64i8)
10924 return Register();
10925 if ((Subtarget->hasAVX512())) {
10926 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10927 }
10928 return Register();
10929}
10930
10931Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10932 if (RetVT.SimpleTy != MVT::v8i16)
10933 return Register();
10934 if ((Subtarget->hasVLX())) {
10935 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10936 }
10937 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10938 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10939 }
10940 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10941 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10942 }
10943 return Register();
10944}
10945
10946Register fastEmit_ISD_AND_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10947 if (RetVT.SimpleTy != MVT::v16i16)
10948 return Register();
10949 if ((Subtarget->hasVLX())) {
10950 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10951 }
10952 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10953 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10954 }
10955 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10956 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10957 }
10958 return Register();
10959}
10960
10961Register fastEmit_ISD_AND_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10962 if (RetVT.SimpleTy != MVT::v32i16)
10963 return Register();
10964 if ((Subtarget->hasAVX512())) {
10965 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10966 }
10967 return Register();
10968}
10969
10970Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10971 if (RetVT.SimpleTy != MVT::v4i32)
10972 return Register();
10973 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10974 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10975 }
10976 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10977 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10978 }
10979 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10980 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10981 }
10982 return Register();
10983}
10984
10985Register fastEmit_ISD_AND_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
10986 if (RetVT.SimpleTy != MVT::v8i32)
10987 return Register();
10988 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10989 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10990 }
10991 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10992 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10993 }
10994 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10995 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10996 }
10997 return Register();
10998}
10999
11000Register fastEmit_ISD_AND_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
11001 if (RetVT.SimpleTy != MVT::v16i32)
11002 return Register();
11003 if ((Subtarget->hasAVX512())) {
11004 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11005 }
11006 return Register();
11007}
11008
11009Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11010 if (RetVT.SimpleTy != MVT::v2i64)
11011 return Register();
11012 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11013 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11014 }
11015 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11016 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
11017 }
11018 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11019 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
11020 }
11021 return Register();
11022}
11023
11024Register fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
11025 if (RetVT.SimpleTy != MVT::v4i64)
11026 return Register();
11027 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
11028 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11029 }
11030 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11031 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11032 }
11033 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11034 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11035 }
11036 return Register();
11037}
11038
11039Register fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
11040 if (RetVT.SimpleTy != MVT::v8i64)
11041 return Register();
11042 if ((Subtarget->hasAVX512())) {
11043 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
11044 }
11045 return Register();
11046}
11047
11048Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11049 switch (VT.SimpleTy) {
11050 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op1);
11051 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op1);
11052 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
11053 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
11054 case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op1);
11055 case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op1);
11056 case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op1);
11057 case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op1);
11058 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
11059 case MVT::v32i8: return fastEmit_ISD_AND_MVT_v32i8_rr(RetVT, Op0, Op1);
11060 case MVT::v64i8: return fastEmit_ISD_AND_MVT_v64i8_rr(RetVT, Op0, Op1);
11061 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
11062 case MVT::v16i16: return fastEmit_ISD_AND_MVT_v16i16_rr(RetVT, Op0, Op1);
11063 case MVT::v32i16: return fastEmit_ISD_AND_MVT_v32i16_rr(RetVT, Op0, Op1);
11064 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
11065 case MVT::v8i32: return fastEmit_ISD_AND_MVT_v8i32_rr(RetVT, Op0, Op1);
11066 case MVT::v16i32: return fastEmit_ISD_AND_MVT_v16i32_rr(RetVT, Op0, Op1);
11067 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
11068 case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op1);
11069 case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op1);
11070 default: return Register();
11071 }
11072}
11073
11074// FastEmit functions for ISD::AVGCEILU.
11075
11076Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11077 if (RetVT.SimpleTy != MVT::v16i8)
11078 return Register();
11079 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11080 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11081 }
11082 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11083 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
11084 }
11085 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11086 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
11087 }
11088 return Register();
11089}
11090
11091Register fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
11092 if (RetVT.SimpleTy != MVT::v32i8)
11093 return Register();
11094 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11095 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11096 }
11097 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11098 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBYrr, RC: &X86::VR256RegClass, Op0, Op1);
11099 }
11100 return Register();
11101}
11102
11103Register fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
11104 if (RetVT.SimpleTy != MVT::v64i8)
11105 return Register();
11106 if ((Subtarget->hasBWI())) {
11107 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZrr, RC: &X86::VR512RegClass, Op0, Op1);
11108 }
11109 return Register();
11110}
11111
11112Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11113 if (RetVT.SimpleTy != MVT::v8i16)
11114 return Register();
11115 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11116 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11117 }
11118 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11119 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
11120 }
11121 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11122 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
11123 }
11124 return Register();
11125}
11126
11127Register fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
11128 if (RetVT.SimpleTy != MVT::v16i16)
11129 return Register();
11130 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11131 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11132 }
11133 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11134 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWYrr, RC: &X86::VR256RegClass, Op0, Op1);
11135 }
11136 return Register();
11137}
11138
11139Register fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
11140 if (RetVT.SimpleTy != MVT::v32i16)
11141 return Register();
11142 if ((Subtarget->hasBWI())) {
11143 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11144 }
11145 return Register();
11146}
11147
11148Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11149 switch (VT.SimpleTy) {
11150 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
11151 case MVT::v32i8: return fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(RetVT, Op0, Op1);
11152 case MVT::v64i8: return fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(RetVT, Op0, Op1);
11153 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
11154 case MVT::v16i16: return fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(RetVT, Op0, Op1);
11155 case MVT::v32i16: return fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(RetVT, Op0, Op1);
11156 default: return Register();
11157 }
11158}
11159
11160// FastEmit functions for ISD::FADD.
11161
11162Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11163 if (RetVT.SimpleTy != MVT::f16)
11164 return Register();
11165 if ((Subtarget->hasFP16())) {
11166 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11167 }
11168 return Register();
11169}
11170
11171Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11172 if (RetVT.SimpleTy != MVT::f32)
11173 return Register();
11174 if ((Subtarget->hasAVX512())) {
11175 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11176 }
11177 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11178 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11179 }
11180 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11181 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11182 }
11183 if ((!Subtarget->hasSSE1())) {
11184 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11185 }
11186 return Register();
11187}
11188
11189Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11190 if (RetVT.SimpleTy != MVT::f64)
11191 return Register();
11192 if ((Subtarget->hasAVX512())) {
11193 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11194 }
11195 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11196 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11197 }
11198 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11199 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11200 }
11201 if ((!Subtarget->hasSSE2())) {
11202 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11203 }
11204 return Register();
11205}
11206
11207Register fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11208 if (RetVT.SimpleTy != MVT::f80)
11209 return Register();
11210 if ((Subtarget->hasX87())) {
11211 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11212 }
11213 return Register();
11214}
11215
11216Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11217 if (RetVT.SimpleTy != MVT::v8f16)
11218 return Register();
11219 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11220 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11221 }
11222 return Register();
11223}
11224
11225Register fastEmit_ISD_FADD_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11226 if (RetVT.SimpleTy != MVT::v16f16)
11227 return Register();
11228 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11229 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11230 }
11231 return Register();
11232}
11233
11234Register fastEmit_ISD_FADD_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11235 if (RetVT.SimpleTy != MVT::v32f16)
11236 return Register();
11237 if ((Subtarget->hasFP16())) {
11238 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11239 }
11240 return Register();
11241}
11242
11243Register fastEmit_ISD_FADD_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11244 if (RetVT.SimpleTy != MVT::v8bf16)
11245 return Register();
11246 if ((Subtarget->hasAVX10_2())) {
11247 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11248 }
11249 return Register();
11250}
11251
11252Register fastEmit_ISD_FADD_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11253 if (RetVT.SimpleTy != MVT::v16bf16)
11254 return Register();
11255 if ((Subtarget->hasAVX10_2())) {
11256 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11257 }
11258 return Register();
11259}
11260
11261Register fastEmit_ISD_FADD_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11262 if (RetVT.SimpleTy != MVT::v32bf16)
11263 return Register();
11264 if ((Subtarget->hasAVX10_2())) {
11265 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11266 }
11267 return Register();
11268}
11269
11270Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11271 if (RetVT.SimpleTy != MVT::v4f32)
11272 return Register();
11273 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11274 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11275 }
11276 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11277 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11278 }
11279 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11280 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11281 }
11282 return Register();
11283}
11284
11285Register fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11286 if (RetVT.SimpleTy != MVT::v8f32)
11287 return Register();
11288 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11289 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11290 }
11291 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11292 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11293 }
11294 return Register();
11295}
11296
11297Register fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11298 if (RetVT.SimpleTy != MVT::v16f32)
11299 return Register();
11300 if ((Subtarget->hasAVX512())) {
11301 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11302 }
11303 return Register();
11304}
11305
11306Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11307 if (RetVT.SimpleTy != MVT::v2f64)
11308 return Register();
11309 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11310 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11311 }
11312 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11313 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11314 }
11315 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11316 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11317 }
11318 return Register();
11319}
11320
11321Register fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11322 if (RetVT.SimpleTy != MVT::v4f64)
11323 return Register();
11324 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11325 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11326 }
11327 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11328 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11329 }
11330 return Register();
11331}
11332
11333Register fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11334 if (RetVT.SimpleTy != MVT::v8f64)
11335 return Register();
11336 if ((Subtarget->hasAVX512())) {
11337 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11338 }
11339 return Register();
11340}
11341
11342Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11343 switch (VT.SimpleTy) {
11344 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11345 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11346 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11347 case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op1);
11348 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11349 case MVT::v16f16: return fastEmit_ISD_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
11350 case MVT::v32f16: return fastEmit_ISD_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
11351 case MVT::v8bf16: return fastEmit_ISD_FADD_MVT_v8bf16_rr(RetVT, Op0, Op1);
11352 case MVT::v16bf16: return fastEmit_ISD_FADD_MVT_v16bf16_rr(RetVT, Op0, Op1);
11353 case MVT::v32bf16: return fastEmit_ISD_FADD_MVT_v32bf16_rr(RetVT, Op0, Op1);
11354 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11355 case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
11356 case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
11357 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11358 case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
11359 case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
11360 default: return Register();
11361 }
11362}
11363
11364// FastEmit functions for ISD::FDIV.
11365
11366Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11367 if (RetVT.SimpleTy != MVT::f16)
11368 return Register();
11369 if ((Subtarget->hasFP16())) {
11370 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11371 }
11372 return Register();
11373}
11374
11375Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11376 if (RetVT.SimpleTy != MVT::f32)
11377 return Register();
11378 if ((Subtarget->hasAVX512())) {
11379 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11380 }
11381 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11382 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11383 }
11384 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11385 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11386 }
11387 if ((!Subtarget->hasSSE1())) {
11388 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11389 }
11390 return Register();
11391}
11392
11393Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11394 if (RetVT.SimpleTy != MVT::f64)
11395 return Register();
11396 if ((Subtarget->hasAVX512())) {
11397 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11398 }
11399 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11400 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11401 }
11402 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11403 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11404 }
11405 if ((!Subtarget->hasSSE2())) {
11406 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11407 }
11408 return Register();
11409}
11410
11411Register fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11412 if (RetVT.SimpleTy != MVT::f80)
11413 return Register();
11414 if ((Subtarget->hasX87())) {
11415 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11416 }
11417 return Register();
11418}
11419
11420Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11421 if (RetVT.SimpleTy != MVT::v8f16)
11422 return Register();
11423 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11424 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11425 }
11426 return Register();
11427}
11428
11429Register fastEmit_ISD_FDIV_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11430 if (RetVT.SimpleTy != MVT::v16f16)
11431 return Register();
11432 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11433 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11434 }
11435 return Register();
11436}
11437
11438Register fastEmit_ISD_FDIV_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11439 if (RetVT.SimpleTy != MVT::v32f16)
11440 return Register();
11441 if ((Subtarget->hasFP16())) {
11442 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11443 }
11444 return Register();
11445}
11446
11447Register fastEmit_ISD_FDIV_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11448 if (RetVT.SimpleTy != MVT::v8bf16)
11449 return Register();
11450 if ((Subtarget->hasAVX10_2())) {
11451 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11452 }
11453 return Register();
11454}
11455
11456Register fastEmit_ISD_FDIV_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11457 if (RetVT.SimpleTy != MVT::v16bf16)
11458 return Register();
11459 if ((Subtarget->hasAVX10_2())) {
11460 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11461 }
11462 return Register();
11463}
11464
11465Register fastEmit_ISD_FDIV_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11466 if (RetVT.SimpleTy != MVT::v32bf16)
11467 return Register();
11468 if ((Subtarget->hasAVX10_2())) {
11469 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11470 }
11471 return Register();
11472}
11473
11474Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11475 if (RetVT.SimpleTy != MVT::v4f32)
11476 return Register();
11477 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11478 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11479 }
11480 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11481 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11482 }
11483 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11484 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11485 }
11486 return Register();
11487}
11488
11489Register fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11490 if (RetVT.SimpleTy != MVT::v8f32)
11491 return Register();
11492 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11493 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11494 }
11495 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11496 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11497 }
11498 return Register();
11499}
11500
11501Register fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11502 if (RetVT.SimpleTy != MVT::v16f32)
11503 return Register();
11504 if ((Subtarget->hasAVX512())) {
11505 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11506 }
11507 return Register();
11508}
11509
11510Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11511 if (RetVT.SimpleTy != MVT::v2f64)
11512 return Register();
11513 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11514 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11515 }
11516 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11517 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11518 }
11519 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11520 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11521 }
11522 return Register();
11523}
11524
11525Register fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11526 if (RetVT.SimpleTy != MVT::v4f64)
11527 return Register();
11528 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11529 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11530 }
11531 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11532 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11533 }
11534 return Register();
11535}
11536
11537Register fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11538 if (RetVT.SimpleTy != MVT::v8f64)
11539 return Register();
11540 if ((Subtarget->hasAVX512())) {
11541 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11542 }
11543 return Register();
11544}
11545
11546Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11547 switch (VT.SimpleTy) {
11548 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11549 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11550 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11551 case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
11552 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11553 case MVT::v16f16: return fastEmit_ISD_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
11554 case MVT::v32f16: return fastEmit_ISD_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
11555 case MVT::v8bf16: return fastEmit_ISD_FDIV_MVT_v8bf16_rr(RetVT, Op0, Op1);
11556 case MVT::v16bf16: return fastEmit_ISD_FDIV_MVT_v16bf16_rr(RetVT, Op0, Op1);
11557 case MVT::v32bf16: return fastEmit_ISD_FDIV_MVT_v32bf16_rr(RetVT, Op0, Op1);
11558 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11559 case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
11560 case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
11561 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11562 case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
11563 case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
11564 default: return Register();
11565 }
11566}
11567
11568// FastEmit functions for ISD::FMUL.
11569
11570Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11571 if (RetVT.SimpleTy != MVT::f16)
11572 return Register();
11573 if ((Subtarget->hasFP16())) {
11574 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11575 }
11576 return Register();
11577}
11578
11579Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11580 if (RetVT.SimpleTy != MVT::f32)
11581 return Register();
11582 if ((Subtarget->hasAVX512())) {
11583 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11584 }
11585 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11586 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11587 }
11588 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11589 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11590 }
11591 if ((!Subtarget->hasSSE1())) {
11592 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11593 }
11594 return Register();
11595}
11596
11597Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11598 if (RetVT.SimpleTy != MVT::f64)
11599 return Register();
11600 if ((Subtarget->hasAVX512())) {
11601 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11602 }
11603 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11604 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11605 }
11606 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11607 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11608 }
11609 if ((!Subtarget->hasSSE2())) {
11610 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11611 }
11612 return Register();
11613}
11614
11615Register fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11616 if (RetVT.SimpleTy != MVT::f80)
11617 return Register();
11618 if ((Subtarget->hasX87())) {
11619 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11620 }
11621 return Register();
11622}
11623
11624Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11625 if (RetVT.SimpleTy != MVT::v8f16)
11626 return Register();
11627 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11628 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11629 }
11630 return Register();
11631}
11632
11633Register fastEmit_ISD_FMUL_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11634 if (RetVT.SimpleTy != MVT::v16f16)
11635 return Register();
11636 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11637 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11638 }
11639 return Register();
11640}
11641
11642Register fastEmit_ISD_FMUL_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11643 if (RetVT.SimpleTy != MVT::v32f16)
11644 return Register();
11645 if ((Subtarget->hasFP16())) {
11646 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11647 }
11648 return Register();
11649}
11650
11651Register fastEmit_ISD_FMUL_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11652 if (RetVT.SimpleTy != MVT::v8bf16)
11653 return Register();
11654 if ((Subtarget->hasAVX10_2())) {
11655 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11656 }
11657 return Register();
11658}
11659
11660Register fastEmit_ISD_FMUL_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11661 if (RetVT.SimpleTy != MVT::v16bf16)
11662 return Register();
11663 if ((Subtarget->hasAVX10_2())) {
11664 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11665 }
11666 return Register();
11667}
11668
11669Register fastEmit_ISD_FMUL_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11670 if (RetVT.SimpleTy != MVT::v32bf16)
11671 return Register();
11672 if ((Subtarget->hasAVX10_2())) {
11673 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11674 }
11675 return Register();
11676}
11677
11678Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11679 if (RetVT.SimpleTy != MVT::v4f32)
11680 return Register();
11681 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11682 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11683 }
11684 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11685 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11686 }
11687 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11688 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11689 }
11690 return Register();
11691}
11692
11693Register fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11694 if (RetVT.SimpleTy != MVT::v8f32)
11695 return Register();
11696 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11697 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11698 }
11699 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11700 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11701 }
11702 return Register();
11703}
11704
11705Register fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11706 if (RetVT.SimpleTy != MVT::v16f32)
11707 return Register();
11708 if ((Subtarget->hasAVX512())) {
11709 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11710 }
11711 return Register();
11712}
11713
11714Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11715 if (RetVT.SimpleTy != MVT::v2f64)
11716 return Register();
11717 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11718 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11719 }
11720 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11721 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11722 }
11723 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11724 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11725 }
11726 return Register();
11727}
11728
11729Register fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11730 if (RetVT.SimpleTy != MVT::v4f64)
11731 return Register();
11732 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11733 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11734 }
11735 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11736 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11737 }
11738 return Register();
11739}
11740
11741Register fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11742 if (RetVT.SimpleTy != MVT::v8f64)
11743 return Register();
11744 if ((Subtarget->hasAVX512())) {
11745 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11746 }
11747 return Register();
11748}
11749
11750Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11751 switch (VT.SimpleTy) {
11752 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11753 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11754 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11755 case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
11756 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11757 case MVT::v16f16: return fastEmit_ISD_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
11758 case MVT::v32f16: return fastEmit_ISD_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
11759 case MVT::v8bf16: return fastEmit_ISD_FMUL_MVT_v8bf16_rr(RetVT, Op0, Op1);
11760 case MVT::v16bf16: return fastEmit_ISD_FMUL_MVT_v16bf16_rr(RetVT, Op0, Op1);
11761 case MVT::v32bf16: return fastEmit_ISD_FMUL_MVT_v32bf16_rr(RetVT, Op0, Op1);
11762 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11763 case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
11764 case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
11765 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11766 case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
11767 case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
11768 default: return Register();
11769 }
11770}
11771
11772// FastEmit functions for ISD::FSUB.
11773
11774Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11775 if (RetVT.SimpleTy != MVT::f16)
11776 return Register();
11777 if ((Subtarget->hasFP16())) {
11778 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11779 }
11780 return Register();
11781}
11782
11783Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11784 if (RetVT.SimpleTy != MVT::f32)
11785 return Register();
11786 if ((Subtarget->hasAVX512())) {
11787 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11788 }
11789 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11790 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11791 }
11792 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11793 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11794 }
11795 if ((!Subtarget->hasSSE1())) {
11796 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11797 }
11798 return Register();
11799}
11800
11801Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11802 if (RetVT.SimpleTy != MVT::f64)
11803 return Register();
11804 if ((Subtarget->hasAVX512())) {
11805 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11806 }
11807 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11808 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11809 }
11810 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11811 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11812 }
11813 if ((!Subtarget->hasSSE2())) {
11814 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11815 }
11816 return Register();
11817}
11818
11819Register fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11820 if (RetVT.SimpleTy != MVT::f80)
11821 return Register();
11822 if ((Subtarget->hasX87())) {
11823 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11824 }
11825 return Register();
11826}
11827
11828Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11829 if (RetVT.SimpleTy != MVT::v8f16)
11830 return Register();
11831 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11832 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11833 }
11834 return Register();
11835}
11836
11837Register fastEmit_ISD_FSUB_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11838 if (RetVT.SimpleTy != MVT::v16f16)
11839 return Register();
11840 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11841 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11842 }
11843 return Register();
11844}
11845
11846Register fastEmit_ISD_FSUB_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11847 if (RetVT.SimpleTy != MVT::v32f16)
11848 return Register();
11849 if ((Subtarget->hasFP16())) {
11850 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11851 }
11852 return Register();
11853}
11854
11855Register fastEmit_ISD_FSUB_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11856 if (RetVT.SimpleTy != MVT::v8bf16)
11857 return Register();
11858 if ((Subtarget->hasAVX10_2())) {
11859 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11860 }
11861 return Register();
11862}
11863
11864Register fastEmit_ISD_FSUB_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11865 if (RetVT.SimpleTy != MVT::v16bf16)
11866 return Register();
11867 if ((Subtarget->hasAVX10_2())) {
11868 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11869 }
11870 return Register();
11871}
11872
11873Register fastEmit_ISD_FSUB_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11874 if (RetVT.SimpleTy != MVT::v32bf16)
11875 return Register();
11876 if ((Subtarget->hasAVX10_2())) {
11877 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11878 }
11879 return Register();
11880}
11881
11882Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11883 if (RetVT.SimpleTy != MVT::v4f32)
11884 return Register();
11885 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11886 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11887 }
11888 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11889 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11890 }
11891 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11892 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11893 }
11894 return Register();
11895}
11896
11897Register fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11898 if (RetVT.SimpleTy != MVT::v8f32)
11899 return Register();
11900 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11901 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11902 }
11903 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11904 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11905 }
11906 return Register();
11907}
11908
11909Register fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11910 if (RetVT.SimpleTy != MVT::v16f32)
11911 return Register();
11912 if ((Subtarget->hasAVX512())) {
11913 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11914 }
11915 return Register();
11916}
11917
11918Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11919 if (RetVT.SimpleTy != MVT::v2f64)
11920 return Register();
11921 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11922 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11923 }
11924 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11925 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11926 }
11927 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11928 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11929 }
11930 return Register();
11931}
11932
11933Register fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11934 if (RetVT.SimpleTy != MVT::v4f64)
11935 return Register();
11936 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11937 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11938 }
11939 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11940 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11941 }
11942 return Register();
11943}
11944
11945Register fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11946 if (RetVT.SimpleTy != MVT::v8f64)
11947 return Register();
11948 if ((Subtarget->hasAVX512())) {
11949 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11950 }
11951 return Register();
11952}
11953
11954Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11955 switch (VT.SimpleTy) {
11956 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11957 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11958 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11959 case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
11960 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11961 case MVT::v16f16: return fastEmit_ISD_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
11962 case MVT::v32f16: return fastEmit_ISD_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
11963 case MVT::v8bf16: return fastEmit_ISD_FSUB_MVT_v8bf16_rr(RetVT, Op0, Op1);
11964 case MVT::v16bf16: return fastEmit_ISD_FSUB_MVT_v16bf16_rr(RetVT, Op0, Op1);
11965 case MVT::v32bf16: return fastEmit_ISD_FSUB_MVT_v32bf16_rr(RetVT, Op0, Op1);
11966 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11967 case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
11968 case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
11969 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11970 case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
11971 case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
11972 default: return Register();
11973 }
11974}
11975
11976// FastEmit functions for ISD::MUL.
11977
11978Register fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
11979 if (RetVT.SimpleTy != MVT::i16)
11980 return Register();
11981 if ((Subtarget->hasNDD())) {
11982 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
11983 }
11984 if ((!Subtarget->hasNDD())) {
11985 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr, RC: &X86::GR16RegClass, Op0, Op1);
11986 }
11987 return Register();
11988}
11989
11990Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11991 if (RetVT.SimpleTy != MVT::i32)
11992 return Register();
11993 if ((Subtarget->hasNDD())) {
11994 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
11995 }
11996 if ((!Subtarget->hasNDD())) {
11997 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr, RC: &X86::GR32RegClass, Op0, Op1);
11998 }
11999 return Register();
12000}
12001
12002Register fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12003 if (RetVT.SimpleTy != MVT::i64)
12004 return Register();
12005 if ((Subtarget->hasNDD())) {
12006 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
12007 }
12008 if ((!Subtarget->hasNDD())) {
12009 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr, RC: &X86::GR64RegClass, Op0, Op1);
12010 }
12011 return Register();
12012}
12013
12014Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12015 if (RetVT.SimpleTy != MVT::v8i16)
12016 return Register();
12017 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12018 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12019 }
12020 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12021 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
12022 }
12023 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12024 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
12025 }
12026 return Register();
12027}
12028
12029Register fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12030 if (RetVT.SimpleTy != MVT::v16i16)
12031 return Register();
12032 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12033 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12034 }
12035 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12036 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12037 }
12038 return Register();
12039}
12040
12041Register fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12042 if (RetVT.SimpleTy != MVT::v32i16)
12043 return Register();
12044 if ((Subtarget->hasBWI())) {
12045 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12046 }
12047 return Register();
12048}
12049
12050Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12051 if (RetVT.SimpleTy != MVT::v4i32)
12052 return Register();
12053 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12054 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12055 }
12056 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12057 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
12058 }
12059 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12060 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
12061 }
12062 return Register();
12063}
12064
12065Register fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12066 if (RetVT.SimpleTy != MVT::v8i32)
12067 return Register();
12068 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12069 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12070 }
12071 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12072 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12073 }
12074 return Register();
12075}
12076
12077Register fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12078 if (RetVT.SimpleTy != MVT::v16i32)
12079 return Register();
12080 if ((Subtarget->hasAVX512())) {
12081 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12082 }
12083 return Register();
12084}
12085
12086Register fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12087 if (RetVT.SimpleTy != MVT::v2i64)
12088 return Register();
12089 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
12090 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12091 }
12092 return Register();
12093}
12094
12095Register fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12096 if (RetVT.SimpleTy != MVT::v4i64)
12097 return Register();
12098 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
12099 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12100 }
12101 return Register();
12102}
12103
12104Register fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12105 if (RetVT.SimpleTy != MVT::v8i64)
12106 return Register();
12107 if ((Subtarget->hasDQI())) {
12108 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12109 }
12110 return Register();
12111}
12112
12113Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12114 switch (VT.SimpleTy) {
12115 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op1);
12116 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
12117 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op1);
12118 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
12119 case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op1);
12120 case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op1);
12121 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
12122 case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op1);
12123 case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op1);
12124 case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op1);
12125 case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op1);
12126 case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op1);
12127 default: return Register();
12128 }
12129}
12130
12131// FastEmit functions for ISD::MULHS.
12132
12133Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12134 if (RetVT.SimpleTy != MVT::v8i16)
12135 return Register();
12136 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12137 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12138 }
12139 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12140 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
12141 }
12142 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12143 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
12144 }
12145 return Register();
12146}
12147
12148Register fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12149 if (RetVT.SimpleTy != MVT::v16i16)
12150 return Register();
12151 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12152 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12153 }
12154 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12155 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12156 }
12157 return Register();
12158}
12159
12160Register fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12161 if (RetVT.SimpleTy != MVT::v32i16)
12162 return Register();
12163 if ((Subtarget->hasBWI())) {
12164 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12165 }
12166 return Register();
12167}
12168
12169Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12170 switch (VT.SimpleTy) {
12171 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
12172 case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op1);
12173 case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op1);
12174 default: return Register();
12175 }
12176}
12177
12178// FastEmit functions for ISD::MULHU.
12179
12180Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12181 if (RetVT.SimpleTy != MVT::v8i16)
12182 return Register();
12183 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12184 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12185 }
12186 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12187 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12188 }
12189 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12190 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12191 }
12192 return Register();
12193}
12194
12195Register fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12196 if (RetVT.SimpleTy != MVT::v16i16)
12197 return Register();
12198 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12199 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12200 }
12201 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12202 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12203 }
12204 return Register();
12205}
12206
12207Register fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12208 if (RetVT.SimpleTy != MVT::v32i16)
12209 return Register();
12210 if ((Subtarget->hasBWI())) {
12211 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12212 }
12213 return Register();
12214}
12215
12216Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12217 switch (VT.SimpleTy) {
12218 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
12219 case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op1);
12220 case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op1);
12221 default: return Register();
12222 }
12223}
12224
12225// FastEmit functions for ISD::OR.
12226
12227Register fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12228 if (RetVT.SimpleTy != MVT::i8)
12229 return Register();
12230 if ((Subtarget->hasNDD())) {
12231 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
12232 }
12233 if ((!Subtarget->hasNDD())) {
12234 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr, RC: &X86::GR8RegClass, Op0, Op1);
12235 }
12236 return Register();
12237}
12238
12239Register fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
12240 if (RetVT.SimpleTy != MVT::i16)
12241 return Register();
12242 if ((Subtarget->hasNDD())) {
12243 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
12244 }
12245 if ((!Subtarget->hasNDD())) {
12246 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr, RC: &X86::GR16RegClass, Op0, Op1);
12247 }
12248 return Register();
12249}
12250
12251Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12252 if (RetVT.SimpleTy != MVT::i32)
12253 return Register();
12254 if ((Subtarget->hasNDD())) {
12255 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
12256 }
12257 if ((!Subtarget->hasNDD())) {
12258 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr, RC: &X86::GR32RegClass, Op0, Op1);
12259 }
12260 return Register();
12261}
12262
12263Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12264 if (RetVT.SimpleTy != MVT::i64)
12265 return Register();
12266 if ((Subtarget->hasNDD())) {
12267 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
12268 }
12269 if ((!Subtarget->hasNDD())) {
12270 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr, RC: &X86::GR64RegClass, Op0, Op1);
12271 }
12272 return Register();
12273}
12274
12275Register fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
12276 if (RetVT.SimpleTy != MVT::v8i1)
12277 return Register();
12278 if ((Subtarget->hasDQI())) {
12279 return fastEmitInst_rr(MachineInstOpcode: X86::KORBkk, RC: &X86::VK8RegClass, Op0, Op1);
12280 }
12281 return Register();
12282}
12283
12284Register fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
12285 if (RetVT.SimpleTy != MVT::v16i1)
12286 return Register();
12287 if ((Subtarget->hasAVX512())) {
12288 return fastEmitInst_rr(MachineInstOpcode: X86::KORWkk, RC: &X86::VK16RegClass, Op0, Op1);
12289 }
12290 return Register();
12291}
12292
12293Register fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
12294 if (RetVT.SimpleTy != MVT::v32i1)
12295 return Register();
12296 if ((Subtarget->hasBWI())) {
12297 return fastEmitInst_rr(MachineInstOpcode: X86::KORDkk, RC: &X86::VK32RegClass, Op0, Op1);
12298 }
12299 return Register();
12300}
12301
12302Register fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
12303 if (RetVT.SimpleTy != MVT::v64i1)
12304 return Register();
12305 if ((Subtarget->hasBWI())) {
12306 return fastEmitInst_rr(MachineInstOpcode: X86::KORQkk, RC: &X86::VK64RegClass, Op0, Op1);
12307 }
12308 return Register();
12309}
12310
12311Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12312 if (RetVT.SimpleTy != MVT::v16i8)
12313 return Register();
12314 if ((Subtarget->hasVLX())) {
12315 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12316 }
12317 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12318 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12319 }
12320 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12321 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12322 }
12323 return Register();
12324}
12325
12326Register fastEmit_ISD_OR_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12327 if (RetVT.SimpleTy != MVT::v32i8)
12328 return Register();
12329 if ((Subtarget->hasVLX())) {
12330 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12331 }
12332 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12333 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12334 }
12335 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12336 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12337 }
12338 return Register();
12339}
12340
12341Register fastEmit_ISD_OR_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12342 if (RetVT.SimpleTy != MVT::v64i8)
12343 return Register();
12344 if ((Subtarget->hasAVX512())) {
12345 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12346 }
12347 return Register();
12348}
12349
12350Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12351 if (RetVT.SimpleTy != MVT::v8i16)
12352 return Register();
12353 if ((Subtarget->hasVLX())) {
12354 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12355 }
12356 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12357 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12358 }
12359 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12360 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12361 }
12362 return Register();
12363}
12364
12365Register fastEmit_ISD_OR_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12366 if (RetVT.SimpleTy != MVT::v16i16)
12367 return Register();
12368 if ((Subtarget->hasVLX())) {
12369 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12370 }
12371 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12372 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12373 }
12374 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12375 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12376 }
12377 return Register();
12378}
12379
12380Register fastEmit_ISD_OR_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12381 if (RetVT.SimpleTy != MVT::v32i16)
12382 return Register();
12383 if ((Subtarget->hasAVX512())) {
12384 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12385 }
12386 return Register();
12387}
12388
12389Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12390 if (RetVT.SimpleTy != MVT::v4i32)
12391 return Register();
12392 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12393 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12394 }
12395 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12396 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12397 }
12398 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12399 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12400 }
12401 return Register();
12402}
12403
12404Register fastEmit_ISD_OR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12405 if (RetVT.SimpleTy != MVT::v8i32)
12406 return Register();
12407 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12408 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12409 }
12410 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12411 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12412 }
12413 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12414 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12415 }
12416 return Register();
12417}
12418
12419Register fastEmit_ISD_OR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12420 if (RetVT.SimpleTy != MVT::v16i32)
12421 return Register();
12422 if ((Subtarget->hasAVX512())) {
12423 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12424 }
12425 return Register();
12426}
12427
12428Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12429 if (RetVT.SimpleTy != MVT::v2i64)
12430 return Register();
12431 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12432 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12433 }
12434 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12435 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12436 }
12437 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12438 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12439 }
12440 return Register();
12441}
12442
12443Register fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12444 if (RetVT.SimpleTy != MVT::v4i64)
12445 return Register();
12446 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12447 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12448 }
12449 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12450 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12451 }
12452 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12453 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12454 }
12455 return Register();
12456}
12457
12458Register fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12459 if (RetVT.SimpleTy != MVT::v8i64)
12460 return Register();
12461 if ((Subtarget->hasAVX512())) {
12462 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12463 }
12464 return Register();
12465}
12466
12467Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12468 switch (VT.SimpleTy) {
12469 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op1);
12470 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op1);
12471 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12472 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12473 case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op1);
12474 case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op1);
12475 case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op1);
12476 case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op1);
12477 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12478 case MVT::v32i8: return fastEmit_ISD_OR_MVT_v32i8_rr(RetVT, Op0, Op1);
12479 case MVT::v64i8: return fastEmit_ISD_OR_MVT_v64i8_rr(RetVT, Op0, Op1);
12480 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12481 case MVT::v16i16: return fastEmit_ISD_OR_MVT_v16i16_rr(RetVT, Op0, Op1);
12482 case MVT::v32i16: return fastEmit_ISD_OR_MVT_v32i16_rr(RetVT, Op0, Op1);
12483 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12484 case MVT::v8i32: return fastEmit_ISD_OR_MVT_v8i32_rr(RetVT, Op0, Op1);
12485 case MVT::v16i32: return fastEmit_ISD_OR_MVT_v16i32_rr(RetVT, Op0, Op1);
12486 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12487 case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op1);
12488 case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op1);
12489 default: return Register();
12490 }
12491}
12492
12493// FastEmit functions for ISD::PDEP.
12494
12495Register fastEmit_ISD_PDEP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12496 if (RetVT.SimpleTy != MVT::i32)
12497 return Register();
12498 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12499 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
12500 }
12501 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12502 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr, RC: &X86::GR32RegClass, Op0, Op1);
12503 }
12504 return Register();
12505}
12506
12507Register fastEmit_ISD_PDEP_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12508 if (RetVT.SimpleTy != MVT::i64)
12509 return Register();
12510 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12511 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
12512 }
12513 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12514 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr, RC: &X86::GR64RegClass, Op0, Op1);
12515 }
12516 return Register();
12517}
12518
12519Register fastEmit_ISD_PDEP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12520 switch (VT.SimpleTy) {
12521 case MVT::i32: return fastEmit_ISD_PDEP_MVT_i32_rr(RetVT, Op0, Op1);
12522 case MVT::i64: return fastEmit_ISD_PDEP_MVT_i64_rr(RetVT, Op0, Op1);
12523 default: return Register();
12524 }
12525}
12526
12527// FastEmit functions for ISD::PEXT.
12528
12529Register fastEmit_ISD_PEXT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12530 if (RetVT.SimpleTy != MVT::i32)
12531 return Register();
12532 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12533 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
12534 }
12535 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12536 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr, RC: &X86::GR32RegClass, Op0, Op1);
12537 }
12538 return Register();
12539}
12540
12541Register fastEmit_ISD_PEXT_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12542 if (RetVT.SimpleTy != MVT::i64)
12543 return Register();
12544 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12545 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
12546 }
12547 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12548 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr, RC: &X86::GR64RegClass, Op0, Op1);
12549 }
12550 return Register();
12551}
12552
12553Register fastEmit_ISD_PEXT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12554 switch (VT.SimpleTy) {
12555 case MVT::i32: return fastEmit_ISD_PEXT_MVT_i32_rr(RetVT, Op0, Op1);
12556 case MVT::i64: return fastEmit_ISD_PEXT_MVT_i64_rr(RetVT, Op0, Op1);
12557 default: return Register();
12558 }
12559}
12560
12561// FastEmit functions for ISD::PSEUDO_FMAX.
12562
12563Register fastEmit_ISD_PSEUDO_FMAX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12564 if (RetVT.SimpleTy != MVT::f16)
12565 return Register();
12566 if ((Subtarget->hasFP16())) {
12567 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
12568 }
12569 return Register();
12570}
12571
12572Register fastEmit_ISD_PSEUDO_FMAX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12573 if (RetVT.SimpleTy != MVT::f32)
12574 return Register();
12575 if ((Subtarget->hasAVX512())) {
12576 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
12577 }
12578 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12579 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
12580 }
12581 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12582 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
12583 }
12584 return Register();
12585}
12586
12587Register fastEmit_ISD_PSEUDO_FMAX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12588 if (RetVT.SimpleTy != MVT::f64)
12589 return Register();
12590 if ((Subtarget->hasAVX512())) {
12591 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
12592 }
12593 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12594 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
12595 }
12596 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12597 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
12598 }
12599 return Register();
12600}
12601
12602Register fastEmit_ISD_PSEUDO_FMAX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12603 if (RetVT.SimpleTy != MVT::v8f16)
12604 return Register();
12605 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
12606 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12607 }
12608 return Register();
12609}
12610
12611Register fastEmit_ISD_PSEUDO_FMAX_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
12612 if (RetVT.SimpleTy != MVT::v16f16)
12613 return Register();
12614 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
12615 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12616 }
12617 return Register();
12618}
12619
12620Register fastEmit_ISD_PSEUDO_FMAX_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
12621 if (RetVT.SimpleTy != MVT::v32f16)
12622 return Register();
12623 if ((Subtarget->hasFP16())) {
12624 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
12625 }
12626 return Register();
12627}
12628
12629Register fastEmit_ISD_PSEUDO_FMAX_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12630 if (RetVT.SimpleTy != MVT::v8bf16)
12631 return Register();
12632 if ((Subtarget->hasAVX10_2())) {
12633 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12634 }
12635 return Register();
12636}
12637
12638Register fastEmit_ISD_PSEUDO_FMAX_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12639 if (RetVT.SimpleTy != MVT::v16bf16)
12640 return Register();
12641 if ((Subtarget->hasAVX10_2())) {
12642 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12643 }
12644 return Register();
12645}
12646
12647Register fastEmit_ISD_PSEUDO_FMAX_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12648 if (RetVT.SimpleTy != MVT::v32bf16)
12649 return Register();
12650 if ((Subtarget->hasAVX10_2())) {
12651 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
12652 }
12653 return Register();
12654}
12655
12656Register fastEmit_ISD_PSEUDO_FMAX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12657 if (RetVT.SimpleTy != MVT::v4f32)
12658 return Register();
12659 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12660 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12661 }
12662 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12663 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
12664 }
12665 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12666 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
12667 }
12668 return Register();
12669}
12670
12671Register fastEmit_ISD_PSEUDO_FMAX_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
12672 if (RetVT.SimpleTy != MVT::v8f32)
12673 return Register();
12674 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12675 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12676 }
12677 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12678 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12679 }
12680 return Register();
12681}
12682
12683Register fastEmit_ISD_PSEUDO_FMAX_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
12684 if (RetVT.SimpleTy != MVT::v16f32)
12685 return Register();
12686 if ((Subtarget->hasAVX512())) {
12687 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
12688 }
12689 return Register();
12690}
12691
12692Register fastEmit_ISD_PSEUDO_FMAX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12693 if (RetVT.SimpleTy != MVT::v2f64)
12694 return Register();
12695 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12696 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12697 }
12698 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12699 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
12700 }
12701 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12702 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
12703 }
12704 return Register();
12705}
12706
12707Register fastEmit_ISD_PSEUDO_FMAX_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
12708 if (RetVT.SimpleTy != MVT::v4f64)
12709 return Register();
12710 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12711 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12712 }
12713 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12714 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12715 }
12716 return Register();
12717}
12718
12719Register fastEmit_ISD_PSEUDO_FMAX_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
12720 if (RetVT.SimpleTy != MVT::v8f64)
12721 return Register();
12722 if ((Subtarget->hasAVX512())) {
12723 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12724 }
12725 return Register();
12726}
12727
12728Register fastEmit_ISD_PSEUDO_FMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12729 switch (VT.SimpleTy) {
12730 case MVT::f16: return fastEmit_ISD_PSEUDO_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
12731 case MVT::f32: return fastEmit_ISD_PSEUDO_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
12732 case MVT::f64: return fastEmit_ISD_PSEUDO_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
12733 case MVT::v8f16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
12734 case MVT::v16f16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
12735 case MVT::v32f16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
12736 case MVT::v8bf16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v8bf16_rr(RetVT, Op0, Op1);
12737 case MVT::v16bf16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v16bf16_rr(RetVT, Op0, Op1);
12738 case MVT::v32bf16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v32bf16_rr(RetVT, Op0, Op1);
12739 case MVT::v4f32: return fastEmit_ISD_PSEUDO_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
12740 case MVT::v8f32: return fastEmit_ISD_PSEUDO_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
12741 case MVT::v16f32: return fastEmit_ISD_PSEUDO_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
12742 case MVT::v2f64: return fastEmit_ISD_PSEUDO_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
12743 case MVT::v4f64: return fastEmit_ISD_PSEUDO_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
12744 case MVT::v8f64: return fastEmit_ISD_PSEUDO_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
12745 default: return Register();
12746 }
12747}
12748
12749// FastEmit functions for ISD::PSEUDO_FMIN.
12750
12751Register fastEmit_ISD_PSEUDO_FMIN_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12752 if (RetVT.SimpleTy != MVT::f16)
12753 return Register();
12754 if ((Subtarget->hasFP16())) {
12755 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
12756 }
12757 return Register();
12758}
12759
12760Register fastEmit_ISD_PSEUDO_FMIN_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12761 if (RetVT.SimpleTy != MVT::f32)
12762 return Register();
12763 if ((Subtarget->hasAVX512())) {
12764 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
12765 }
12766 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12767 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
12768 }
12769 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12770 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
12771 }
12772 return Register();
12773}
12774
12775Register fastEmit_ISD_PSEUDO_FMIN_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12776 if (RetVT.SimpleTy != MVT::f64)
12777 return Register();
12778 if ((Subtarget->hasAVX512())) {
12779 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
12780 }
12781 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12782 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
12783 }
12784 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12785 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
12786 }
12787 return Register();
12788}
12789
12790Register fastEmit_ISD_PSEUDO_FMIN_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12791 if (RetVT.SimpleTy != MVT::v8f16)
12792 return Register();
12793 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
12794 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12795 }
12796 return Register();
12797}
12798
12799Register fastEmit_ISD_PSEUDO_FMIN_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
12800 if (RetVT.SimpleTy != MVT::v16f16)
12801 return Register();
12802 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
12803 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12804 }
12805 return Register();
12806}
12807
12808Register fastEmit_ISD_PSEUDO_FMIN_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
12809 if (RetVT.SimpleTy != MVT::v32f16)
12810 return Register();
12811 if ((Subtarget->hasFP16())) {
12812 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
12813 }
12814 return Register();
12815}
12816
12817Register fastEmit_ISD_PSEUDO_FMIN_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12818 if (RetVT.SimpleTy != MVT::v8bf16)
12819 return Register();
12820 if ((Subtarget->hasAVX10_2())) {
12821 return fastEmitInst_rr(MachineInstOpcode: X86::VMINBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12822 }
12823 return Register();
12824}
12825
12826Register fastEmit_ISD_PSEUDO_FMIN_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12827 if (RetVT.SimpleTy != MVT::v16bf16)
12828 return Register();
12829 if ((Subtarget->hasAVX10_2())) {
12830 return fastEmitInst_rr(MachineInstOpcode: X86::VMINBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12831 }
12832 return Register();
12833}
12834
12835Register fastEmit_ISD_PSEUDO_FMIN_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12836 if (RetVT.SimpleTy != MVT::v32bf16)
12837 return Register();
12838 if ((Subtarget->hasAVX10_2())) {
12839 return fastEmitInst_rr(MachineInstOpcode: X86::VMINBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
12840 }
12841 return Register();
12842}
12843
12844Register fastEmit_ISD_PSEUDO_FMIN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12845 if (RetVT.SimpleTy != MVT::v4f32)
12846 return Register();
12847 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12848 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12849 }
12850 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12851 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
12852 }
12853 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12854 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
12855 }
12856 return Register();
12857}
12858
12859Register fastEmit_ISD_PSEUDO_FMIN_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
12860 if (RetVT.SimpleTy != MVT::v8f32)
12861 return Register();
12862 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12863 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12864 }
12865 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12866 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12867 }
12868 return Register();
12869}
12870
12871Register fastEmit_ISD_PSEUDO_FMIN_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
12872 if (RetVT.SimpleTy != MVT::v16f32)
12873 return Register();
12874 if ((Subtarget->hasAVX512())) {
12875 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
12876 }
12877 return Register();
12878}
12879
12880Register fastEmit_ISD_PSEUDO_FMIN_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12881 if (RetVT.SimpleTy != MVT::v2f64)
12882 return Register();
12883 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12884 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12885 }
12886 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12887 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
12888 }
12889 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12890 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
12891 }
12892 return Register();
12893}
12894
12895Register fastEmit_ISD_PSEUDO_FMIN_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
12896 if (RetVT.SimpleTy != MVT::v4f64)
12897 return Register();
12898 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12899 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12900 }
12901 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12902 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12903 }
12904 return Register();
12905}
12906
12907Register fastEmit_ISD_PSEUDO_FMIN_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
12908 if (RetVT.SimpleTy != MVT::v8f64)
12909 return Register();
12910 if ((Subtarget->hasAVX512())) {
12911 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12912 }
12913 return Register();
12914}
12915
12916Register fastEmit_ISD_PSEUDO_FMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12917 switch (VT.SimpleTy) {
12918 case MVT::f16: return fastEmit_ISD_PSEUDO_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
12919 case MVT::f32: return fastEmit_ISD_PSEUDO_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
12920 case MVT::f64: return fastEmit_ISD_PSEUDO_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
12921 case MVT::v8f16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
12922 case MVT::v16f16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
12923 case MVT::v32f16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
12924 case MVT::v8bf16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v8bf16_rr(RetVT, Op0, Op1);
12925 case MVT::v16bf16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v16bf16_rr(RetVT, Op0, Op1);
12926 case MVT::v32bf16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v32bf16_rr(RetVT, Op0, Op1);
12927 case MVT::v4f32: return fastEmit_ISD_PSEUDO_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
12928 case MVT::v8f32: return fastEmit_ISD_PSEUDO_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
12929 case MVT::v16f32: return fastEmit_ISD_PSEUDO_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
12930 case MVT::v2f64: return fastEmit_ISD_PSEUDO_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
12931 case MVT::v4f64: return fastEmit_ISD_PSEUDO_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
12932 case MVT::v8f64: return fastEmit_ISD_PSEUDO_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
12933 default: return Register();
12934 }
12935}
12936
12937// FastEmit functions for ISD::ROTL.
12938
12939Register fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12940 if (RetVT.SimpleTy != MVT::i8)
12941 return Register();
12942 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12943 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12944 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL_ND, RC: &X86::GR8RegClass, Op0);
12945 }
12946 if ((!Subtarget->hasNDD())) {
12947 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12948 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL, RC: &X86::GR8RegClass, Op0);
12949 }
12950 return Register();
12951}
12952
12953Register fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12954 if (RetVT.SimpleTy != MVT::v16i8)
12955 return Register();
12956 if ((Subtarget->hasXOP())) {
12957 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTBrr, RC: &X86::VR128RegClass, Op0, Op1);
12958 }
12959 return Register();
12960}
12961
12962Register fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12963 if (RetVT.SimpleTy != MVT::v8i16)
12964 return Register();
12965 if ((Subtarget->hasXOP())) {
12966 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTWrr, RC: &X86::VR128RegClass, Op0, Op1);
12967 }
12968 return Register();
12969}
12970
12971Register fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12972 if (RetVT.SimpleTy != MVT::v4i32)
12973 return Register();
12974 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12975 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12976 }
12977 if ((Subtarget->hasXOP())) {
12978 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTDrr, RC: &X86::VR128RegClass, Op0, Op1);
12979 }
12980 return Register();
12981}
12982
12983Register fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12984 if (RetVT.SimpleTy != MVT::v8i32)
12985 return Register();
12986 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12987 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12988 }
12989 return Register();
12990}
12991
12992Register fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12993 if (RetVT.SimpleTy != MVT::v16i32)
12994 return Register();
12995 if ((Subtarget->hasAVX512())) {
12996 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12997 }
12998 return Register();
12999}
13000
13001Register fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13002 if (RetVT.SimpleTy != MVT::v2i64)
13003 return Register();
13004 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13005 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13006 }
13007 if ((Subtarget->hasXOP())) {
13008 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTQrr, RC: &X86::VR128RegClass, Op0, Op1);
13009 }
13010 return Register();
13011}
13012
13013Register fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
13014 if (RetVT.SimpleTy != MVT::v4i64)
13015 return Register();
13016 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13017 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13018 }
13019 return Register();
13020}
13021
13022Register fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
13023 if (RetVT.SimpleTy != MVT::v8i64)
13024 return Register();
13025 if ((Subtarget->hasAVX512())) {
13026 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13027 }
13028 return Register();
13029}
13030
13031Register fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13032 switch (VT.SimpleTy) {
13033 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op1);
13034 case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op1);
13035 case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op1);
13036 case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op1);
13037 case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op1);
13038 case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op1);
13039 case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op1);
13040 case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op1);
13041 case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op1);
13042 default: return Register();
13043 }
13044}
13045
13046// FastEmit functions for ISD::ROTR.
13047
13048Register fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13049 if (RetVT.SimpleTy != MVT::i8)
13050 return Register();
13051 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
13052 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13053 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL_ND, RC: &X86::GR8RegClass, Op0);
13054 }
13055 if ((!Subtarget->hasNDD())) {
13056 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13057 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL, RC: &X86::GR8RegClass, Op0);
13058 }
13059 return Register();
13060}
13061
13062Register fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13063 if (RetVT.SimpleTy != MVT::v4i32)
13064 return Register();
13065 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13066 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13067 }
13068 return Register();
13069}
13070
13071Register fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
13072 if (RetVT.SimpleTy != MVT::v8i32)
13073 return Register();
13074 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13075 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13076 }
13077 return Register();
13078}
13079
13080Register fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
13081 if (RetVT.SimpleTy != MVT::v16i32)
13082 return Register();
13083 if ((Subtarget->hasAVX512())) {
13084 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13085 }
13086 return Register();
13087}
13088
13089Register fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13090 if (RetVT.SimpleTy != MVT::v2i64)
13091 return Register();
13092 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13093 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13094 }
13095 return Register();
13096}
13097
13098Register fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
13099 if (RetVT.SimpleTy != MVT::v4i64)
13100 return Register();
13101 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13102 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13103 }
13104 return Register();
13105}
13106
13107Register fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
13108 if (RetVT.SimpleTy != MVT::v8i64)
13109 return Register();
13110 if ((Subtarget->hasAVX512())) {
13111 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13112 }
13113 return Register();
13114}
13115
13116Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13117 switch (VT.SimpleTy) {
13118 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op1);
13119 case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op1);
13120 case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op1);
13121 case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op1);
13122 case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op1);
13123 case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op1);
13124 case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op1);
13125 default: return Register();
13126 }
13127}
13128
13129// FastEmit functions for ISD::SADDSAT.
13130
13131Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13132 if (RetVT.SimpleTy != MVT::v16i8)
13133 return Register();
13134 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13135 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13136 }
13137 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13138 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13139 }
13140 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13141 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13142 }
13143 return Register();
13144}
13145
13146Register fastEmit_ISD_SADDSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13147 if (RetVT.SimpleTy != MVT::v32i8)
13148 return Register();
13149 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13150 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13151 }
13152 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13153 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13154 }
13155 return Register();
13156}
13157
13158Register fastEmit_ISD_SADDSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13159 if (RetVT.SimpleTy != MVT::v64i8)
13160 return Register();
13161 if ((Subtarget->hasBWI())) {
13162 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13163 }
13164 return Register();
13165}
13166
13167Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13168 if (RetVT.SimpleTy != MVT::v8i16)
13169 return Register();
13170 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13171 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13172 }
13173 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13174 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13175 }
13176 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13177 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13178 }
13179 return Register();
13180}
13181
13182Register fastEmit_ISD_SADDSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13183 if (RetVT.SimpleTy != MVT::v16i16)
13184 return Register();
13185 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13186 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13187 }
13188 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13189 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13190 }
13191 return Register();
13192}
13193
13194Register fastEmit_ISD_SADDSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13195 if (RetVT.SimpleTy != MVT::v32i16)
13196 return Register();
13197 if ((Subtarget->hasBWI())) {
13198 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13199 }
13200 return Register();
13201}
13202
13203Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13204 switch (VT.SimpleTy) {
13205 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13206 case MVT::v32i8: return fastEmit_ISD_SADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
13207 case MVT::v64i8: return fastEmit_ISD_SADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
13208 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13209 case MVT::v16i16: return fastEmit_ISD_SADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
13210 case MVT::v32i16: return fastEmit_ISD_SADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
13211 default: return Register();
13212 }
13213}
13214
13215// FastEmit functions for ISD::SHL.
13216
13217Register fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13218 if (RetVT.SimpleTy != MVT::i8)
13219 return Register();
13220 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
13221 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13222 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL_ND, RC: &X86::GR8RegClass, Op0);
13223 }
13224 if ((!Subtarget->hasNDD())) {
13225 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13226 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL, RC: &X86::GR8RegClass, Op0);
13227 }
13228 return Register();
13229}
13230
13231Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13232 switch (VT.SimpleTy) {
13233 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op1);
13234 default: return Register();
13235 }
13236}
13237
13238// FastEmit functions for ISD::SMAX.
13239
13240Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13241 if (RetVT.SimpleTy != MVT::v16i8)
13242 return Register();
13243 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13244 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13245 }
13246 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13247 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13248 }
13249 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13250 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13251 }
13252 return Register();
13253}
13254
13255Register fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13256 if (RetVT.SimpleTy != MVT::v32i8)
13257 return Register();
13258 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13259 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13260 }
13261 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13262 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13263 }
13264 return Register();
13265}
13266
13267Register fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13268 if (RetVT.SimpleTy != MVT::v64i8)
13269 return Register();
13270 if ((Subtarget->hasBWI())) {
13271 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13272 }
13273 return Register();
13274}
13275
13276Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13277 if (RetVT.SimpleTy != MVT::v8i16)
13278 return Register();
13279 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13280 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13281 }
13282 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13283 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13284 }
13285 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13286 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13287 }
13288 return Register();
13289}
13290
13291Register fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13292 if (RetVT.SimpleTy != MVT::v16i16)
13293 return Register();
13294 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13295 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13296 }
13297 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13298 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13299 }
13300 return Register();
13301}
13302
13303Register fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13304 if (RetVT.SimpleTy != MVT::v32i16)
13305 return Register();
13306 if ((Subtarget->hasBWI())) {
13307 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13308 }
13309 return Register();
13310}
13311
13312Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13313 if (RetVT.SimpleTy != MVT::v4i32)
13314 return Register();
13315 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13316 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13317 }
13318 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13319 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
13320 }
13321 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13322 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
13323 }
13324 return Register();
13325}
13326
13327Register fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
13328 if (RetVT.SimpleTy != MVT::v8i32)
13329 return Register();
13330 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13331 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13332 }
13333 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13334 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13335 }
13336 return Register();
13337}
13338
13339Register fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
13340 if (RetVT.SimpleTy != MVT::v16i32)
13341 return Register();
13342 if ((Subtarget->hasAVX512())) {
13343 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13344 }
13345 return Register();
13346}
13347
13348Register fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13349 if (RetVT.SimpleTy != MVT::v2i64)
13350 return Register();
13351 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13352 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13353 }
13354 return Register();
13355}
13356
13357Register fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
13358 if (RetVT.SimpleTy != MVT::v4i64)
13359 return Register();
13360 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13361 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13362 }
13363 return Register();
13364}
13365
13366Register fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
13367 if (RetVT.SimpleTy != MVT::v8i64)
13368 return Register();
13369 if ((Subtarget->hasAVX512())) {
13370 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13371 }
13372 return Register();
13373}
13374
13375Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13376 switch (VT.SimpleTy) {
13377 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13378 case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
13379 case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
13380 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13381 case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
13382 case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
13383 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13384 case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
13385 case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
13386 case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
13387 case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
13388 case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
13389 default: return Register();
13390 }
13391}
13392
13393// FastEmit functions for ISD::SMIN.
13394
13395Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13396 if (RetVT.SimpleTy != MVT::v16i8)
13397 return Register();
13398 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13399 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13400 }
13401 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13402 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13403 }
13404 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13405 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13406 }
13407 return Register();
13408}
13409
13410Register fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13411 if (RetVT.SimpleTy != MVT::v32i8)
13412 return Register();
13413 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13414 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13415 }
13416 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13417 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13418 }
13419 return Register();
13420}
13421
13422Register fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13423 if (RetVT.SimpleTy != MVT::v64i8)
13424 return Register();
13425 if ((Subtarget->hasBWI())) {
13426 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13427 }
13428 return Register();
13429}
13430
13431Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13432 if (RetVT.SimpleTy != MVT::v8i16)
13433 return Register();
13434 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13435 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13436 }
13437 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13438 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13439 }
13440 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13441 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13442 }
13443 return Register();
13444}
13445
13446Register fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13447 if (RetVT.SimpleTy != MVT::v16i16)
13448 return Register();
13449 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13450 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13451 }
13452 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13453 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13454 }
13455 return Register();
13456}
13457
13458Register fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13459 if (RetVT.SimpleTy != MVT::v32i16)
13460 return Register();
13461 if ((Subtarget->hasBWI())) {
13462 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13463 }
13464 return Register();
13465}
13466
13467Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13468 if (RetVT.SimpleTy != MVT::v4i32)
13469 return Register();
13470 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13471 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13472 }
13473 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13474 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
13475 }
13476 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13477 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
13478 }
13479 return Register();
13480}
13481
13482Register fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
13483 if (RetVT.SimpleTy != MVT::v8i32)
13484 return Register();
13485 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13486 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13487 }
13488 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13489 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13490 }
13491 return Register();
13492}
13493
13494Register fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
13495 if (RetVT.SimpleTy != MVT::v16i32)
13496 return Register();
13497 if ((Subtarget->hasAVX512())) {
13498 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13499 }
13500 return Register();
13501}
13502
13503Register fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13504 if (RetVT.SimpleTy != MVT::v2i64)
13505 return Register();
13506 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13507 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13508 }
13509 return Register();
13510}
13511
13512Register fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
13513 if (RetVT.SimpleTy != MVT::v4i64)
13514 return Register();
13515 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13516 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13517 }
13518 return Register();
13519}
13520
13521Register fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
13522 if (RetVT.SimpleTy != MVT::v8i64)
13523 return Register();
13524 if ((Subtarget->hasAVX512())) {
13525 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13526 }
13527 return Register();
13528}
13529
13530Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13531 switch (VT.SimpleTy) {
13532 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13533 case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
13534 case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
13535 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13536 case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
13537 case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
13538 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13539 case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
13540 case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
13541 case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
13542 case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
13543 case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
13544 default: return Register();
13545 }
13546}
13547
13548// FastEmit functions for ISD::SRA.
13549
13550Register fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13551 if (RetVT.SimpleTy != MVT::i8)
13552 return Register();
13553 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
13554 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13555 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL_ND, RC: &X86::GR8RegClass, Op0);
13556 }
13557 if ((!Subtarget->hasNDD())) {
13558 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13559 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL, RC: &X86::GR8RegClass, Op0);
13560 }
13561 return Register();
13562}
13563
13564Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13565 switch (VT.SimpleTy) {
13566 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op1);
13567 default: return Register();
13568 }
13569}
13570
13571// FastEmit functions for ISD::SRL.
13572
13573Register fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13574 if (RetVT.SimpleTy != MVT::i8)
13575 return Register();
13576 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
13577 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13578 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL_ND, RC: &X86::GR8RegClass, Op0);
13579 }
13580 if ((!Subtarget->hasNDD())) {
13581 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13582 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL, RC: &X86::GR8RegClass, Op0);
13583 }
13584 return Register();
13585}
13586
13587Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13588 switch (VT.SimpleTy) {
13589 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op1);
13590 default: return Register();
13591 }
13592}
13593
13594// FastEmit functions for ISD::SSUBSAT.
13595
13596Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13597 if (RetVT.SimpleTy != MVT::v16i8)
13598 return Register();
13599 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13600 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13601 }
13602 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13603 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13604 }
13605 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13606 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13607 }
13608 return Register();
13609}
13610
13611Register fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13612 if (RetVT.SimpleTy != MVT::v32i8)
13613 return Register();
13614 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13615 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13616 }
13617 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13618 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13619 }
13620 return Register();
13621}
13622
13623Register fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13624 if (RetVT.SimpleTy != MVT::v64i8)
13625 return Register();
13626 if ((Subtarget->hasBWI())) {
13627 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13628 }
13629 return Register();
13630}
13631
13632Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13633 if (RetVT.SimpleTy != MVT::v8i16)
13634 return Register();
13635 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13636 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13637 }
13638 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13639 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13640 }
13641 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13642 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13643 }
13644 return Register();
13645}
13646
13647Register fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13648 if (RetVT.SimpleTy != MVT::v16i16)
13649 return Register();
13650 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13651 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13652 }
13653 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13654 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13655 }
13656 return Register();
13657}
13658
13659Register fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13660 if (RetVT.SimpleTy != MVT::v32i16)
13661 return Register();
13662 if ((Subtarget->hasBWI())) {
13663 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13664 }
13665 return Register();
13666}
13667
13668Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13669 switch (VT.SimpleTy) {
13670 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13671 case MVT::v32i8: return fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
13672 case MVT::v64i8: return fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
13673 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13674 case MVT::v16i16: return fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
13675 case MVT::v32i16: return fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
13676 default: return Register();
13677 }
13678}
13679
13680// FastEmit functions for ISD::STRICT_FADD.
13681
13682Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13683 if (RetVT.SimpleTy != MVT::f16)
13684 return Register();
13685 if ((Subtarget->hasFP16())) {
13686 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13687 }
13688 return Register();
13689}
13690
13691Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13692 if (RetVT.SimpleTy != MVT::f32)
13693 return Register();
13694 if ((Subtarget->hasAVX512())) {
13695 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13696 }
13697 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13698 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13699 }
13700 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13701 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13702 }
13703 if ((!Subtarget->hasSSE1())) {
13704 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13705 }
13706 return Register();
13707}
13708
13709Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13710 if (RetVT.SimpleTy != MVT::f64)
13711 return Register();
13712 if ((Subtarget->hasAVX512())) {
13713 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13714 }
13715 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13716 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13717 }
13718 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13719 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13720 }
13721 if ((!Subtarget->hasSSE2())) {
13722 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13723 }
13724 return Register();
13725}
13726
13727Register fastEmit_ISD_STRICT_FADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13728 if (RetVT.SimpleTy != MVT::f80)
13729 return Register();
13730 if ((Subtarget->hasX87())) {
13731 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13732 }
13733 return Register();
13734}
13735
13736Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13737 if (RetVT.SimpleTy != MVT::v8f16)
13738 return Register();
13739 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13740 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13741 }
13742 return Register();
13743}
13744
13745Register fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13746 if (RetVT.SimpleTy != MVT::v16f16)
13747 return Register();
13748 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13749 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13750 }
13751 return Register();
13752}
13753
13754Register fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13755 if (RetVT.SimpleTy != MVT::v32f16)
13756 return Register();
13757 if ((Subtarget->hasFP16())) {
13758 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13759 }
13760 return Register();
13761}
13762
13763Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13764 if (RetVT.SimpleTy != MVT::v4f32)
13765 return Register();
13766 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13767 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13768 }
13769 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13770 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13771 }
13772 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13773 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13774 }
13775 return Register();
13776}
13777
13778Register fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13779 if (RetVT.SimpleTy != MVT::v8f32)
13780 return Register();
13781 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13782 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13783 }
13784 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13785 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13786 }
13787 return Register();
13788}
13789
13790Register fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13791 if (RetVT.SimpleTy != MVT::v16f32)
13792 return Register();
13793 if ((Subtarget->hasAVX512())) {
13794 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13795 }
13796 return Register();
13797}
13798
13799Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13800 if (RetVT.SimpleTy != MVT::v2f64)
13801 return Register();
13802 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13803 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13804 }
13805 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13806 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13807 }
13808 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13809 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13810 }
13811 return Register();
13812}
13813
13814Register fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13815 if (RetVT.SimpleTy != MVT::v4f64)
13816 return Register();
13817 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13818 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13819 }
13820 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13821 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13822 }
13823 return Register();
13824}
13825
13826Register fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13827 if (RetVT.SimpleTy != MVT::v8f64)
13828 return Register();
13829 if ((Subtarget->hasAVX512())) {
13830 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13831 }
13832 return Register();
13833}
13834
13835Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13836 switch (VT.SimpleTy) {
13837 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
13838 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
13839 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
13840 case MVT::f80: return fastEmit_ISD_STRICT_FADD_MVT_f80_rr(RetVT, Op0, Op1);
13841 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
13842 case MVT::v16f16: return fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
13843 case MVT::v32f16: return fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
13844 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
13845 case MVT::v8f32: return fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
13846 case MVT::v16f32: return fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
13847 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
13848 case MVT::v4f64: return fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
13849 case MVT::v8f64: return fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
13850 default: return Register();
13851 }
13852}
13853
13854// FastEmit functions for ISD::STRICT_FDIV.
13855
13856Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13857 if (RetVT.SimpleTy != MVT::f16)
13858 return Register();
13859 if ((Subtarget->hasFP16())) {
13860 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13861 }
13862 return Register();
13863}
13864
13865Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13866 if (RetVT.SimpleTy != MVT::f32)
13867 return Register();
13868 if ((Subtarget->hasAVX512())) {
13869 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13870 }
13871 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13872 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13873 }
13874 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13875 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13876 }
13877 if ((!Subtarget->hasSSE1())) {
13878 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13879 }
13880 return Register();
13881}
13882
13883Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13884 if (RetVT.SimpleTy != MVT::f64)
13885 return Register();
13886 if ((Subtarget->hasAVX512())) {
13887 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13888 }
13889 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13890 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13891 }
13892 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13893 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13894 }
13895 if ((!Subtarget->hasSSE2())) {
13896 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13897 }
13898 return Register();
13899}
13900
13901Register fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13902 if (RetVT.SimpleTy != MVT::f80)
13903 return Register();
13904 if ((Subtarget->hasX87())) {
13905 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13906 }
13907 return Register();
13908}
13909
13910Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13911 if (RetVT.SimpleTy != MVT::v8f16)
13912 return Register();
13913 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13914 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13915 }
13916 return Register();
13917}
13918
13919Register fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13920 if (RetVT.SimpleTy != MVT::v16f16)
13921 return Register();
13922 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13923 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13924 }
13925 return Register();
13926}
13927
13928Register fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13929 if (RetVT.SimpleTy != MVT::v32f16)
13930 return Register();
13931 if ((Subtarget->hasFP16())) {
13932 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13933 }
13934 return Register();
13935}
13936
13937Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13938 if (RetVT.SimpleTy != MVT::v4f32)
13939 return Register();
13940 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13941 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13942 }
13943 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13944 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13945 }
13946 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13947 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13948 }
13949 return Register();
13950}
13951
13952Register fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13953 if (RetVT.SimpleTy != MVT::v8f32)
13954 return Register();
13955 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13956 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13957 }
13958 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13959 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13960 }
13961 return Register();
13962}
13963
13964Register fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13965 if (RetVT.SimpleTy != MVT::v16f32)
13966 return Register();
13967 if ((Subtarget->hasAVX512())) {
13968 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13969 }
13970 return Register();
13971}
13972
13973Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13974 if (RetVT.SimpleTy != MVT::v2f64)
13975 return Register();
13976 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13977 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13978 }
13979 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13980 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13981 }
13982 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13983 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13984 }
13985 return Register();
13986}
13987
13988Register fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13989 if (RetVT.SimpleTy != MVT::v4f64)
13990 return Register();
13991 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13992 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13993 }
13994 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13995 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13996 }
13997 return Register();
13998}
13999
14000Register fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
14001 if (RetVT.SimpleTy != MVT::v8f64)
14002 return Register();
14003 if ((Subtarget->hasAVX512())) {
14004 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14005 }
14006 return Register();
14007}
14008
14009Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14010 switch (VT.SimpleTy) {
14011 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
14012 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
14013 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
14014 case MVT::f80: return fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
14015 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
14016 case MVT::v16f16: return fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
14017 case MVT::v32f16: return fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
14018 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
14019 case MVT::v8f32: return fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
14020 case MVT::v16f32: return fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
14021 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
14022 case MVT::v4f64: return fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
14023 case MVT::v8f64: return fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
14024 default: return Register();
14025 }
14026}
14027
14028// FastEmit functions for ISD::STRICT_FMUL.
14029
14030Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
14031 if (RetVT.SimpleTy != MVT::f16)
14032 return Register();
14033 if ((Subtarget->hasFP16())) {
14034 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
14035 }
14036 return Register();
14037}
14038
14039Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
14040 if (RetVT.SimpleTy != MVT::f32)
14041 return Register();
14042 if ((Subtarget->hasAVX512())) {
14043 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
14044 }
14045 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14046 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14047 }
14048 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14049 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14050 }
14051 if ((!Subtarget->hasSSE1())) {
14052 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
14053 }
14054 return Register();
14055}
14056
14057Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
14058 if (RetVT.SimpleTy != MVT::f64)
14059 return Register();
14060 if ((Subtarget->hasAVX512())) {
14061 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
14062 }
14063 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14064 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14065 }
14066 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14067 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14068 }
14069 if ((!Subtarget->hasSSE2())) {
14070 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
14071 }
14072 return Register();
14073}
14074
14075Register fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
14076 if (RetVT.SimpleTy != MVT::f80)
14077 return Register();
14078 if ((Subtarget->hasX87())) {
14079 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
14080 }
14081 return Register();
14082}
14083
14084Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
14085 if (RetVT.SimpleTy != MVT::v8f16)
14086 return Register();
14087 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14088 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14089 }
14090 return Register();
14091}
14092
14093Register fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
14094 if (RetVT.SimpleTy != MVT::v16f16)
14095 return Register();
14096 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14097 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14098 }
14099 return Register();
14100}
14101
14102Register fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
14103 if (RetVT.SimpleTy != MVT::v32f16)
14104 return Register();
14105 if ((Subtarget->hasFP16())) {
14106 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
14107 }
14108 return Register();
14109}
14110
14111Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
14112 if (RetVT.SimpleTy != MVT::v4f32)
14113 return Register();
14114 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14115 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14116 }
14117 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14118 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14119 }
14120 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14121 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14122 }
14123 return Register();
14124}
14125
14126Register fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
14127 if (RetVT.SimpleTy != MVT::v8f32)
14128 return Register();
14129 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14130 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14131 }
14132 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14133 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14134 }
14135 return Register();
14136}
14137
14138Register fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
14139 if (RetVT.SimpleTy != MVT::v16f32)
14140 return Register();
14141 if ((Subtarget->hasAVX512())) {
14142 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14143 }
14144 return Register();
14145}
14146
14147Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14148 if (RetVT.SimpleTy != MVT::v2f64)
14149 return Register();
14150 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14151 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14152 }
14153 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14154 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14155 }
14156 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14157 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14158 }
14159 return Register();
14160}
14161
14162Register fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14163 if (RetVT.SimpleTy != MVT::v4f64)
14164 return Register();
14165 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14166 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14167 }
14168 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14169 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14170 }
14171 return Register();
14172}
14173
14174Register fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
14175 if (RetVT.SimpleTy != MVT::v8f64)
14176 return Register();
14177 if ((Subtarget->hasAVX512())) {
14178 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14179 }
14180 return Register();
14181}
14182
14183Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14184 switch (VT.SimpleTy) {
14185 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
14186 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
14187 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
14188 case MVT::f80: return fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
14189 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
14190 case MVT::v16f16: return fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
14191 case MVT::v32f16: return fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
14192 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
14193 case MVT::v8f32: return fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
14194 case MVT::v16f32: return fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
14195 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
14196 case MVT::v4f64: return fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
14197 case MVT::v8f64: return fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
14198 default: return Register();
14199 }
14200}
14201
14202// FastEmit functions for ISD::STRICT_FSUB.
14203
14204Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
14205 if (RetVT.SimpleTy != MVT::f16)
14206 return Register();
14207 if ((Subtarget->hasFP16())) {
14208 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
14209 }
14210 return Register();
14211}
14212
14213Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
14214 if (RetVT.SimpleTy != MVT::f32)
14215 return Register();
14216 if ((Subtarget->hasAVX512())) {
14217 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
14218 }
14219 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14220 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14221 }
14222 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14223 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14224 }
14225 if ((!Subtarget->hasSSE1())) {
14226 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
14227 }
14228 return Register();
14229}
14230
14231Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
14232 if (RetVT.SimpleTy != MVT::f64)
14233 return Register();
14234 if ((Subtarget->hasAVX512())) {
14235 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
14236 }
14237 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14238 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14239 }
14240 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14241 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14242 }
14243 if ((!Subtarget->hasSSE2())) {
14244 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
14245 }
14246 return Register();
14247}
14248
14249Register fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
14250 if (RetVT.SimpleTy != MVT::f80)
14251 return Register();
14252 if ((Subtarget->hasX87())) {
14253 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
14254 }
14255 return Register();
14256}
14257
14258Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
14259 if (RetVT.SimpleTy != MVT::v8f16)
14260 return Register();
14261 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14262 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14263 }
14264 return Register();
14265}
14266
14267Register fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
14268 if (RetVT.SimpleTy != MVT::v16f16)
14269 return Register();
14270 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14271 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14272 }
14273 return Register();
14274}
14275
14276Register fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
14277 if (RetVT.SimpleTy != MVT::v32f16)
14278 return Register();
14279 if ((Subtarget->hasFP16())) {
14280 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
14281 }
14282 return Register();
14283}
14284
14285Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
14286 if (RetVT.SimpleTy != MVT::v4f32)
14287 return Register();
14288 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14289 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14290 }
14291 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14292 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14293 }
14294 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14295 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14296 }
14297 return Register();
14298}
14299
14300Register fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
14301 if (RetVT.SimpleTy != MVT::v8f32)
14302 return Register();
14303 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14304 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14305 }
14306 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14307 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14308 }
14309 return Register();
14310}
14311
14312Register fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
14313 if (RetVT.SimpleTy != MVT::v16f32)
14314 return Register();
14315 if ((Subtarget->hasAVX512())) {
14316 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14317 }
14318 return Register();
14319}
14320
14321Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14322 if (RetVT.SimpleTy != MVT::v2f64)
14323 return Register();
14324 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14325 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14326 }
14327 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14328 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14329 }
14330 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14331 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14332 }
14333 return Register();
14334}
14335
14336Register fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14337 if (RetVT.SimpleTy != MVT::v4f64)
14338 return Register();
14339 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14340 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14341 }
14342 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14343 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14344 }
14345 return Register();
14346}
14347
14348Register fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
14349 if (RetVT.SimpleTy != MVT::v8f64)
14350 return Register();
14351 if ((Subtarget->hasAVX512())) {
14352 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14353 }
14354 return Register();
14355}
14356
14357Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14358 switch (VT.SimpleTy) {
14359 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
14360 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
14361 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
14362 case MVT::f80: return fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
14363 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
14364 case MVT::v16f16: return fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
14365 case MVT::v32f16: return fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
14366 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
14367 case MVT::v8f32: return fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
14368 case MVT::v16f32: return fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
14369 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
14370 case MVT::v4f64: return fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
14371 case MVT::v8f64: return fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
14372 default: return Register();
14373 }
14374}
14375
14376// FastEmit functions for ISD::STRICT_PSEUDO_FMAX.
14377
14378Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
14379 if (RetVT.SimpleTy != MVT::f16)
14380 return Register();
14381 if ((Subtarget->hasFP16())) {
14382 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
14383 }
14384 return Register();
14385}
14386
14387Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
14388 if (RetVT.SimpleTy != MVT::f32)
14389 return Register();
14390 if ((Subtarget->hasAVX512())) {
14391 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
14392 }
14393 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14394 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14395 }
14396 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14397 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14398 }
14399 return Register();
14400}
14401
14402Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
14403 if (RetVT.SimpleTy != MVT::f64)
14404 return Register();
14405 if ((Subtarget->hasAVX512())) {
14406 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
14407 }
14408 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14409 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14410 }
14411 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14412 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14413 }
14414 return Register();
14415}
14416
14417Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
14418 if (RetVT.SimpleTy != MVT::v8f16)
14419 return Register();
14420 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14421 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14422 }
14423 return Register();
14424}
14425
14426Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
14427 if (RetVT.SimpleTy != MVT::v16f16)
14428 return Register();
14429 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14430 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14431 }
14432 return Register();
14433}
14434
14435Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
14436 if (RetVT.SimpleTy != MVT::v32f16)
14437 return Register();
14438 if ((Subtarget->hasFP16())) {
14439 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
14440 }
14441 return Register();
14442}
14443
14444Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
14445 if (RetVT.SimpleTy != MVT::v4f32)
14446 return Register();
14447 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14448 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14449 }
14450 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14451 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14452 }
14453 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14454 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14455 }
14456 return Register();
14457}
14458
14459Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
14460 if (RetVT.SimpleTy != MVT::v8f32)
14461 return Register();
14462 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14463 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14464 }
14465 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14466 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14467 }
14468 return Register();
14469}
14470
14471Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
14472 if (RetVT.SimpleTy != MVT::v16f32)
14473 return Register();
14474 if ((Subtarget->hasAVX512())) {
14475 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14476 }
14477 return Register();
14478}
14479
14480Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14481 if (RetVT.SimpleTy != MVT::v2f64)
14482 return Register();
14483 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14484 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14485 }
14486 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14487 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14488 }
14489 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14490 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14491 }
14492 return Register();
14493}
14494
14495Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14496 if (RetVT.SimpleTy != MVT::v4f64)
14497 return Register();
14498 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14499 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14500 }
14501 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14502 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14503 }
14504 return Register();
14505}
14506
14507Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
14508 if (RetVT.SimpleTy != MVT::v8f64)
14509 return Register();
14510 if ((Subtarget->hasAVX512())) {
14511 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14512 }
14513 return Register();
14514}
14515
14516Register fastEmit_ISD_STRICT_PSEUDO_FMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14517 switch (VT.SimpleTy) {
14518 case MVT::f16: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
14519 case MVT::f32: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
14520 case MVT::f64: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
14521 case MVT::v8f16: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
14522 case MVT::v16f16: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
14523 case MVT::v32f16: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
14524 case MVT::v4f32: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
14525 case MVT::v8f32: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
14526 case MVT::v16f32: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
14527 case MVT::v2f64: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
14528 case MVT::v4f64: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
14529 case MVT::v8f64: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
14530 default: return Register();
14531 }
14532}
14533
14534// FastEmit functions for ISD::STRICT_PSEUDO_FMIN.
14535
14536Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
14537 if (RetVT.SimpleTy != MVT::f16)
14538 return Register();
14539 if ((Subtarget->hasFP16())) {
14540 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
14541 }
14542 return Register();
14543}
14544
14545Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
14546 if (RetVT.SimpleTy != MVT::f32)
14547 return Register();
14548 if ((Subtarget->hasAVX512())) {
14549 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
14550 }
14551 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14552 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14553 }
14554 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14555 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14556 }
14557 return Register();
14558}
14559
14560Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
14561 if (RetVT.SimpleTy != MVT::f64)
14562 return Register();
14563 if ((Subtarget->hasAVX512())) {
14564 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
14565 }
14566 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14567 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14568 }
14569 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14570 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14571 }
14572 return Register();
14573}
14574
14575Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
14576 if (RetVT.SimpleTy != MVT::v8f16)
14577 return Register();
14578 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14579 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14580 }
14581 return Register();
14582}
14583
14584Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
14585 if (RetVT.SimpleTy != MVT::v16f16)
14586 return Register();
14587 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14588 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14589 }
14590 return Register();
14591}
14592
14593Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
14594 if (RetVT.SimpleTy != MVT::v32f16)
14595 return Register();
14596 if ((Subtarget->hasFP16())) {
14597 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
14598 }
14599 return Register();
14600}
14601
14602Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
14603 if (RetVT.SimpleTy != MVT::v4f32)
14604 return Register();
14605 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14606 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14607 }
14608 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14609 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14610 }
14611 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14612 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14613 }
14614 return Register();
14615}
14616
14617Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
14618 if (RetVT.SimpleTy != MVT::v8f32)
14619 return Register();
14620 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14621 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14622 }
14623 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14624 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14625 }
14626 return Register();
14627}
14628
14629Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
14630 if (RetVT.SimpleTy != MVT::v16f32)
14631 return Register();
14632 if ((Subtarget->hasAVX512())) {
14633 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14634 }
14635 return Register();
14636}
14637
14638Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14639 if (RetVT.SimpleTy != MVT::v2f64)
14640 return Register();
14641 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14642 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14643 }
14644 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14645 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14646 }
14647 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14648 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14649 }
14650 return Register();
14651}
14652
14653Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14654 if (RetVT.SimpleTy != MVT::v4f64)
14655 return Register();
14656 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14657 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14658 }
14659 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14660 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14661 }
14662 return Register();
14663}
14664
14665Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
14666 if (RetVT.SimpleTy != MVT::v8f64)
14667 return Register();
14668 if ((Subtarget->hasAVX512())) {
14669 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14670 }
14671 return Register();
14672}
14673
14674Register fastEmit_ISD_STRICT_PSEUDO_FMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14675 switch (VT.SimpleTy) {
14676 case MVT::f16: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
14677 case MVT::f32: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
14678 case MVT::f64: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
14679 case MVT::v8f16: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
14680 case MVT::v16f16: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
14681 case MVT::v32f16: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
14682 case MVT::v4f32: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
14683 case MVT::v8f32: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
14684 case MVT::v16f32: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
14685 case MVT::v2f64: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
14686 case MVT::v4f64: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
14687 case MVT::v8f64: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
14688 default: return Register();
14689 }
14690}
14691
14692// FastEmit functions for ISD::SUB.
14693
14694Register fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
14695 if (RetVT.SimpleTy != MVT::i8)
14696 return Register();
14697 if ((Subtarget->hasNDD())) {
14698 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
14699 }
14700 if ((!Subtarget->hasNDD())) {
14701 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr, RC: &X86::GR8RegClass, Op0, Op1);
14702 }
14703 return Register();
14704}
14705
14706Register fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
14707 if (RetVT.SimpleTy != MVT::i16)
14708 return Register();
14709 if ((Subtarget->hasNDD())) {
14710 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
14711 }
14712 if ((!Subtarget->hasNDD())) {
14713 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr, RC: &X86::GR16RegClass, Op0, Op1);
14714 }
14715 return Register();
14716}
14717
14718Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14719 if (RetVT.SimpleTy != MVT::i32)
14720 return Register();
14721 if ((Subtarget->hasNDD())) {
14722 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
14723 }
14724 if ((!Subtarget->hasNDD())) {
14725 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr, RC: &X86::GR32RegClass, Op0, Op1);
14726 }
14727 return Register();
14728}
14729
14730Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14731 if (RetVT.SimpleTy != MVT::i64)
14732 return Register();
14733 if ((Subtarget->hasNDD())) {
14734 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
14735 }
14736 if ((!Subtarget->hasNDD())) {
14737 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr, RC: &X86::GR64RegClass, Op0, Op1);
14738 }
14739 return Register();
14740}
14741
14742Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14743 if (RetVT.SimpleTy != MVT::v16i8)
14744 return Register();
14745 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14746 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14747 }
14748 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14749 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
14750 }
14751 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14752 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
14753 }
14754 return Register();
14755}
14756
14757Register fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14758 if (RetVT.SimpleTy != MVT::v32i8)
14759 return Register();
14760 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14761 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14762 }
14763 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14764 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14765 }
14766 return Register();
14767}
14768
14769Register fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14770 if (RetVT.SimpleTy != MVT::v64i8)
14771 return Register();
14772 if ((Subtarget->hasBWI())) {
14773 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14774 }
14775 return Register();
14776}
14777
14778Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14779 if (RetVT.SimpleTy != MVT::v8i16)
14780 return Register();
14781 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14782 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14783 }
14784 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14785 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
14786 }
14787 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14788 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
14789 }
14790 return Register();
14791}
14792
14793Register fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14794 if (RetVT.SimpleTy != MVT::v16i16)
14795 return Register();
14796 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14797 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14798 }
14799 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14800 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14801 }
14802 return Register();
14803}
14804
14805Register fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14806 if (RetVT.SimpleTy != MVT::v32i16)
14807 return Register();
14808 if ((Subtarget->hasBWI())) {
14809 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14810 }
14811 return Register();
14812}
14813
14814Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14815 if (RetVT.SimpleTy != MVT::v4i32)
14816 return Register();
14817 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14818 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14819 }
14820 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14821 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
14822 }
14823 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14824 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
14825 }
14826 return Register();
14827}
14828
14829Register fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14830 if (RetVT.SimpleTy != MVT::v8i32)
14831 return Register();
14832 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14833 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14834 }
14835 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14836 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14837 }
14838 return Register();
14839}
14840
14841Register fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14842 if (RetVT.SimpleTy != MVT::v16i32)
14843 return Register();
14844 if ((Subtarget->hasAVX512())) {
14845 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14846 }
14847 return Register();
14848}
14849
14850Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14851 if (RetVT.SimpleTy != MVT::v2i64)
14852 return Register();
14853 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14854 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14855 }
14856 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14857 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
14858 }
14859 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14860 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
14861 }
14862 return Register();
14863}
14864
14865Register fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14866 if (RetVT.SimpleTy != MVT::v4i64)
14867 return Register();
14868 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14869 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14870 }
14871 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14872 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQYrr, RC: &X86::VR256RegClass, Op0, Op1);
14873 }
14874 return Register();
14875}
14876
14877Register fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14878 if (RetVT.SimpleTy != MVT::v8i64)
14879 return Register();
14880 if ((Subtarget->hasAVX512())) {
14881 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14882 }
14883 return Register();
14884}
14885
14886Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14887 switch (VT.SimpleTy) {
14888 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op1);
14889 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op1);
14890 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
14891 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
14892 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
14893 case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op1);
14894 case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op1);
14895 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
14896 case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op1);
14897 case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op1);
14898 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
14899 case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op1);
14900 case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op1);
14901 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
14902 case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op1);
14903 case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op1);
14904 default: return Register();
14905 }
14906}
14907
14908// FastEmit functions for ISD::UADDSAT.
14909
14910Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14911 if (RetVT.SimpleTy != MVT::v16i8)
14912 return Register();
14913 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14914 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14915 }
14916 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14917 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14918 }
14919 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14920 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14921 }
14922 return Register();
14923}
14924
14925Register fastEmit_ISD_UADDSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14926 if (RetVT.SimpleTy != MVT::v32i8)
14927 return Register();
14928 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14929 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14930 }
14931 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14932 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14933 }
14934 return Register();
14935}
14936
14937Register fastEmit_ISD_UADDSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14938 if (RetVT.SimpleTy != MVT::v64i8)
14939 return Register();
14940 if ((Subtarget->hasBWI())) {
14941 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14942 }
14943 return Register();
14944}
14945
14946Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14947 if (RetVT.SimpleTy != MVT::v8i16)
14948 return Register();
14949 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14950 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14951 }
14952 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14953 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14954 }
14955 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14956 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14957 }
14958 return Register();
14959}
14960
14961Register fastEmit_ISD_UADDSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14962 if (RetVT.SimpleTy != MVT::v16i16)
14963 return Register();
14964 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14965 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14966 }
14967 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14968 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14969 }
14970 return Register();
14971}
14972
14973Register fastEmit_ISD_UADDSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14974 if (RetVT.SimpleTy != MVT::v32i16)
14975 return Register();
14976 if ((Subtarget->hasBWI())) {
14977 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14978 }
14979 return Register();
14980}
14981
14982Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14983 switch (VT.SimpleTy) {
14984 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14985 case MVT::v32i8: return fastEmit_ISD_UADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
14986 case MVT::v64i8: return fastEmit_ISD_UADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
14987 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14988 case MVT::v16i16: return fastEmit_ISD_UADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
14989 case MVT::v32i16: return fastEmit_ISD_UADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
14990 default: return Register();
14991 }
14992}
14993
14994// FastEmit functions for ISD::UMAX.
14995
14996Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14997 if (RetVT.SimpleTy != MVT::v16i8)
14998 return Register();
14999 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15000 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15001 }
15002 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15003 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
15004 }
15005 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15006 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
15007 }
15008 return Register();
15009}
15010
15011Register fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
15012 if (RetVT.SimpleTy != MVT::v32i8)
15013 return Register();
15014 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15015 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15016 }
15017 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15018 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
15019 }
15020 return Register();
15021}
15022
15023Register fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
15024 if (RetVT.SimpleTy != MVT::v64i8)
15025 return Register();
15026 if ((Subtarget->hasBWI())) {
15027 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
15028 }
15029 return Register();
15030}
15031
15032Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
15033 if (RetVT.SimpleTy != MVT::v8i16)
15034 return Register();
15035 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15036 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15037 }
15038 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15039 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
15040 }
15041 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15042 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
15043 }
15044 return Register();
15045}
15046
15047Register fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
15048 if (RetVT.SimpleTy != MVT::v16i16)
15049 return Register();
15050 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15051 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15052 }
15053 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15054 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15055 }
15056 return Register();
15057}
15058
15059Register fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
15060 if (RetVT.SimpleTy != MVT::v32i16)
15061 return Register();
15062 if ((Subtarget->hasBWI())) {
15063 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
15064 }
15065 return Register();
15066}
15067
15068Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
15069 if (RetVT.SimpleTy != MVT::v4i32)
15070 return Register();
15071 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15072 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15073 }
15074 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15075 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
15076 }
15077 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15078 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
15079 }
15080 return Register();
15081}
15082
15083Register fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
15084 if (RetVT.SimpleTy != MVT::v8i32)
15085 return Register();
15086 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15087 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15088 }
15089 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15090 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15091 }
15092 return Register();
15093}
15094
15095Register fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
15096 if (RetVT.SimpleTy != MVT::v16i32)
15097 return Register();
15098 if ((Subtarget->hasAVX512())) {
15099 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15100 }
15101 return Register();
15102}
15103
15104Register fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
15105 if (RetVT.SimpleTy != MVT::v2i64)
15106 return Register();
15107 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15108 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15109 }
15110 return Register();
15111}
15112
15113Register fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
15114 if (RetVT.SimpleTy != MVT::v4i64)
15115 return Register();
15116 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15117 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15118 }
15119 return Register();
15120}
15121
15122Register fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
15123 if (RetVT.SimpleTy != MVT::v8i64)
15124 return Register();
15125 if ((Subtarget->hasAVX512())) {
15126 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15127 }
15128 return Register();
15129}
15130
15131Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15132 switch (VT.SimpleTy) {
15133 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
15134 case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
15135 case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
15136 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
15137 case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
15138 case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
15139 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
15140 case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
15141 case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
15142 case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
15143 case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
15144 case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
15145 default: return Register();
15146 }
15147}
15148
15149// FastEmit functions for ISD::UMIN.
15150
15151Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
15152 if (RetVT.SimpleTy != MVT::v16i8)
15153 return Register();
15154 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15155 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15156 }
15157 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15158 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
15159 }
15160 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15161 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
15162 }
15163 return Register();
15164}
15165
15166Register fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
15167 if (RetVT.SimpleTy != MVT::v32i8)
15168 return Register();
15169 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15170 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15171 }
15172 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15173 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
15174 }
15175 return Register();
15176}
15177
15178Register fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
15179 if (RetVT.SimpleTy != MVT::v64i8)
15180 return Register();
15181 if ((Subtarget->hasBWI())) {
15182 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
15183 }
15184 return Register();
15185}
15186
15187Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
15188 if (RetVT.SimpleTy != MVT::v8i16)
15189 return Register();
15190 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15191 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15192 }
15193 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15194 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
15195 }
15196 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15197 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
15198 }
15199 return Register();
15200}
15201
15202Register fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
15203 if (RetVT.SimpleTy != MVT::v16i16)
15204 return Register();
15205 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15206 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15207 }
15208 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15209 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15210 }
15211 return Register();
15212}
15213
15214Register fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
15215 if (RetVT.SimpleTy != MVT::v32i16)
15216 return Register();
15217 if ((Subtarget->hasBWI())) {
15218 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
15219 }
15220 return Register();
15221}
15222
15223Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
15224 if (RetVT.SimpleTy != MVT::v4i32)
15225 return Register();
15226 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15227 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15228 }
15229 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15230 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
15231 }
15232 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15233 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
15234 }
15235 return Register();
15236}
15237
15238Register fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
15239 if (RetVT.SimpleTy != MVT::v8i32)
15240 return Register();
15241 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15242 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15243 }
15244 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15245 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15246 }
15247 return Register();
15248}
15249
15250Register fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
15251 if (RetVT.SimpleTy != MVT::v16i32)
15252 return Register();
15253 if ((Subtarget->hasAVX512())) {
15254 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15255 }
15256 return Register();
15257}
15258
15259Register fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
15260 if (RetVT.SimpleTy != MVT::v2i64)
15261 return Register();
15262 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15263 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15264 }
15265 return Register();
15266}
15267
15268Register fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
15269 if (RetVT.SimpleTy != MVT::v4i64)
15270 return Register();
15271 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15272 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15273 }
15274 return Register();
15275}
15276
15277Register fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
15278 if (RetVT.SimpleTy != MVT::v8i64)
15279 return Register();
15280 if ((Subtarget->hasAVX512())) {
15281 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15282 }
15283 return Register();
15284}
15285
15286Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15287 switch (VT.SimpleTy) {
15288 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
15289 case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
15290 case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
15291 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
15292 case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
15293 case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
15294 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
15295 case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
15296 case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
15297 case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
15298 case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
15299 case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
15300 default: return Register();
15301 }
15302}
15303
15304// FastEmit functions for ISD::USUBSAT.
15305
15306Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
15307 if (RetVT.SimpleTy != MVT::v16i8)
15308 return Register();
15309 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15310 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15311 }
15312 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15313 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
15314 }
15315 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15316 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
15317 }
15318 return Register();
15319}
15320
15321Register fastEmit_ISD_USUBSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
15322 if (RetVT.SimpleTy != MVT::v32i8)
15323 return Register();
15324 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15325 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15326 }
15327 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15328 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
15329 }
15330 return Register();
15331}
15332
15333Register fastEmit_ISD_USUBSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
15334 if (RetVT.SimpleTy != MVT::v64i8)
15335 return Register();
15336 if ((Subtarget->hasBWI())) {
15337 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
15338 }
15339 return Register();
15340}
15341
15342Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
15343 if (RetVT.SimpleTy != MVT::v8i16)
15344 return Register();
15345 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15346 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15347 }
15348 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15349 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
15350 }
15351 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15352 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
15353 }
15354 return Register();
15355}
15356
15357Register fastEmit_ISD_USUBSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
15358 if (RetVT.SimpleTy != MVT::v16i16)
15359 return Register();
15360 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15361 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15362 }
15363 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15364 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15365 }
15366 return Register();
15367}
15368
15369Register fastEmit_ISD_USUBSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
15370 if (RetVT.SimpleTy != MVT::v32i16)
15371 return Register();
15372 if ((Subtarget->hasBWI())) {
15373 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
15374 }
15375 return Register();
15376}
15377
15378Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15379 switch (VT.SimpleTy) {
15380 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
15381 case MVT::v32i8: return fastEmit_ISD_USUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
15382 case MVT::v64i8: return fastEmit_ISD_USUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
15383 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
15384 case MVT::v16i16: return fastEmit_ISD_USUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
15385 case MVT::v32i16: return fastEmit_ISD_USUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
15386 default: return Register();
15387 }
15388}
15389
15390// FastEmit functions for ISD::XOR.
15391
15392Register fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
15393 if (RetVT.SimpleTy != MVT::i8)
15394 return Register();
15395 if ((Subtarget->hasNDD())) {
15396 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
15397 }
15398 if ((!Subtarget->hasNDD())) {
15399 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr, RC: &X86::GR8RegClass, Op0, Op1);
15400 }
15401 return Register();
15402}
15403
15404Register fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
15405 if (RetVT.SimpleTy != MVT::i16)
15406 return Register();
15407 if ((Subtarget->hasNDD())) {
15408 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
15409 }
15410 if ((!Subtarget->hasNDD())) {
15411 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr, RC: &X86::GR16RegClass, Op0, Op1);
15412 }
15413 return Register();
15414}
15415
15416Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15417 if (RetVT.SimpleTy != MVT::i32)
15418 return Register();
15419 if ((Subtarget->hasNDD())) {
15420 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
15421 }
15422 if ((!Subtarget->hasNDD())) {
15423 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr, RC: &X86::GR32RegClass, Op0, Op1);
15424 }
15425 return Register();
15426}
15427
15428Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15429 if (RetVT.SimpleTy != MVT::i64)
15430 return Register();
15431 if ((Subtarget->hasNDD())) {
15432 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
15433 }
15434 if ((!Subtarget->hasNDD())) {
15435 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr, RC: &X86::GR64RegClass, Op0, Op1);
15436 }
15437 return Register();
15438}
15439
15440Register fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
15441 if (RetVT.SimpleTy != MVT::v8i1)
15442 return Register();
15443 if ((Subtarget->hasDQI())) {
15444 return fastEmitInst_rr(MachineInstOpcode: X86::KXORBkk, RC: &X86::VK8RegClass, Op0, Op1);
15445 }
15446 return Register();
15447}
15448
15449Register fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
15450 if (RetVT.SimpleTy != MVT::v16i1)
15451 return Register();
15452 if ((Subtarget->hasAVX512())) {
15453 return fastEmitInst_rr(MachineInstOpcode: X86::KXORWkk, RC: &X86::VK16RegClass, Op0, Op1);
15454 }
15455 return Register();
15456}
15457
15458Register fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
15459 if (RetVT.SimpleTy != MVT::v32i1)
15460 return Register();
15461 if ((Subtarget->hasBWI())) {
15462 return fastEmitInst_rr(MachineInstOpcode: X86::KXORDkk, RC: &X86::VK32RegClass, Op0, Op1);
15463 }
15464 return Register();
15465}
15466
15467Register fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
15468 if (RetVT.SimpleTy != MVT::v64i1)
15469 return Register();
15470 if ((Subtarget->hasBWI())) {
15471 return fastEmitInst_rr(MachineInstOpcode: X86::KXORQkk, RC: &X86::VK64RegClass, Op0, Op1);
15472 }
15473 return Register();
15474}
15475
15476Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
15477 if (RetVT.SimpleTy != MVT::v16i8)
15478 return Register();
15479 if ((Subtarget->hasVLX())) {
15480 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15481 }
15482 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15483 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15484 }
15485 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15486 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15487 }
15488 return Register();
15489}
15490
15491Register fastEmit_ISD_XOR_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
15492 if (RetVT.SimpleTy != MVT::v32i8)
15493 return Register();
15494 if ((Subtarget->hasVLX())) {
15495 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15496 }
15497 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15498 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15499 }
15500 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15501 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
15502 }
15503 return Register();
15504}
15505
15506Register fastEmit_ISD_XOR_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
15507 if (RetVT.SimpleTy != MVT::v64i8)
15508 return Register();
15509 if ((Subtarget->hasAVX512())) {
15510 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15511 }
15512 return Register();
15513}
15514
15515Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
15516 if (RetVT.SimpleTy != MVT::v8i16)
15517 return Register();
15518 if ((Subtarget->hasVLX())) {
15519 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15520 }
15521 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15522 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15523 }
15524 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15525 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15526 }
15527 return Register();
15528}
15529
15530Register fastEmit_ISD_XOR_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
15531 if (RetVT.SimpleTy != MVT::v16i16)
15532 return Register();
15533 if ((Subtarget->hasVLX())) {
15534 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15535 }
15536 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15537 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15538 }
15539 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15540 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
15541 }
15542 return Register();
15543}
15544
15545Register fastEmit_ISD_XOR_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
15546 if (RetVT.SimpleTy != MVT::v32i16)
15547 return Register();
15548 if ((Subtarget->hasAVX512())) {
15549 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15550 }
15551 return Register();
15552}
15553
15554Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
15555 if (RetVT.SimpleTy != MVT::v4i32)
15556 return Register();
15557 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15558 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15559 }
15560 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15561 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15562 }
15563 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15564 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15565 }
15566 return Register();
15567}
15568
15569Register fastEmit_ISD_XOR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
15570 if (RetVT.SimpleTy != MVT::v8i32)
15571 return Register();
15572 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15573 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15574 }
15575 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15576 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
15577 }
15578 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15579 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15580 }
15581 return Register();
15582}
15583
15584Register fastEmit_ISD_XOR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
15585 if (RetVT.SimpleTy != MVT::v16i32)
15586 return Register();
15587 if ((Subtarget->hasAVX512())) {
15588 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15589 }
15590 return Register();
15591}
15592
15593Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
15594 if (RetVT.SimpleTy != MVT::v2i64)
15595 return Register();
15596 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15597 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15598 }
15599 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15600 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15601 }
15602 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15603 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15604 }
15605 return Register();
15606}
15607
15608Register fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
15609 if (RetVT.SimpleTy != MVT::v4i64)
15610 return Register();
15611 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15612 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15613 }
15614 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15615 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15616 }
15617 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15618 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
15619 }
15620 return Register();
15621}
15622
15623Register fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
15624 if (RetVT.SimpleTy != MVT::v8i64)
15625 return Register();
15626 if ((Subtarget->hasAVX512())) {
15627 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15628 }
15629 return Register();
15630}
15631
15632Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15633 switch (VT.SimpleTy) {
15634 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op1);
15635 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op1);
15636 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
15637 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
15638 case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op1);
15639 case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op1);
15640 case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op1);
15641 case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op1);
15642 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
15643 case MVT::v32i8: return fastEmit_ISD_XOR_MVT_v32i8_rr(RetVT, Op0, Op1);
15644 case MVT::v64i8: return fastEmit_ISD_XOR_MVT_v64i8_rr(RetVT, Op0, Op1);
15645 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
15646 case MVT::v16i16: return fastEmit_ISD_XOR_MVT_v16i16_rr(RetVT, Op0, Op1);
15647 case MVT::v32i16: return fastEmit_ISD_XOR_MVT_v32i16_rr(RetVT, Op0, Op1);
15648 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
15649 case MVT::v8i32: return fastEmit_ISD_XOR_MVT_v8i32_rr(RetVT, Op0, Op1);
15650 case MVT::v16i32: return fastEmit_ISD_XOR_MVT_v16i32_rr(RetVT, Op0, Op1);
15651 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
15652 case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op1);
15653 case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op1);
15654 default: return Register();
15655 }
15656}
15657
15658// FastEmit functions for X86ISD::ADDSUB.
15659
15660Register fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15661 if (RetVT.SimpleTy != MVT::v4f32)
15662 return Register();
15663 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15664 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15665 }
15666 if ((Subtarget->hasAVX())) {
15667 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15668 }
15669 return Register();
15670}
15671
15672Register fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15673 if (RetVT.SimpleTy != MVT::v8f32)
15674 return Register();
15675 if ((Subtarget->hasAVX())) {
15676 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15677 }
15678 return Register();
15679}
15680
15681Register fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15682 if (RetVT.SimpleTy != MVT::v2f64)
15683 return Register();
15684 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15685 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15686 }
15687 if ((Subtarget->hasAVX())) {
15688 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15689 }
15690 return Register();
15691}
15692
15693Register fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15694 if (RetVT.SimpleTy != MVT::v4f64)
15695 return Register();
15696 if ((Subtarget->hasAVX())) {
15697 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15698 }
15699 return Register();
15700}
15701
15702Register fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15703 switch (VT.SimpleTy) {
15704 case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
15705 case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
15706 case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
15707 case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
15708 default: return Register();
15709 }
15710}
15711
15712// FastEmit functions for X86ISD::ANDNP.
15713
15714Register fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
15715 if (RetVT.SimpleTy != MVT::v16i8)
15716 return Register();
15717 if ((Subtarget->hasVLX())) {
15718 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15719 }
15720 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15721 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15722 }
15723 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15724 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15725 }
15726 return Register();
15727}
15728
15729Register fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
15730 if (RetVT.SimpleTy != MVT::v32i8)
15731 return Register();
15732 if ((Subtarget->hasVLX())) {
15733 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15734 }
15735 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15736 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15737 }
15738 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15739 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
15740 }
15741 return Register();
15742}
15743
15744Register fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
15745 if (RetVT.SimpleTy != MVT::v64i8)
15746 return Register();
15747 if ((Subtarget->hasAVX512())) {
15748 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15749 }
15750 return Register();
15751}
15752
15753Register fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
15754 if (RetVT.SimpleTy != MVT::v8i16)
15755 return Register();
15756 if ((Subtarget->hasVLX())) {
15757 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15758 }
15759 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15760 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15761 }
15762 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15763 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15764 }
15765 return Register();
15766}
15767
15768Register fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
15769 if (RetVT.SimpleTy != MVT::v16i16)
15770 return Register();
15771 if ((Subtarget->hasVLX())) {
15772 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15773 }
15774 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15775 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15776 }
15777 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15778 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
15779 }
15780 return Register();
15781}
15782
15783Register fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
15784 if (RetVT.SimpleTy != MVT::v32i16)
15785 return Register();
15786 if ((Subtarget->hasAVX512())) {
15787 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15788 }
15789 return Register();
15790}
15791
15792Register fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
15793 if (RetVT.SimpleTy != MVT::v4i32)
15794 return Register();
15795 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15796 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15797 }
15798 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15799 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15800 }
15801 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15802 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15803 }
15804 return Register();
15805}
15806
15807Register fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
15808 if (RetVT.SimpleTy != MVT::v8i32)
15809 return Register();
15810 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15811 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15812 }
15813 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15814 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
15815 }
15816 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15817 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15818 }
15819 return Register();
15820}
15821
15822Register fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
15823 if (RetVT.SimpleTy != MVT::v16i32)
15824 return Register();
15825 if ((Subtarget->hasAVX512())) {
15826 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15827 }
15828 return Register();
15829}
15830
15831Register fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
15832 if (RetVT.SimpleTy != MVT::v2i64)
15833 return Register();
15834 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15835 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15836 }
15837 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15838 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15839 }
15840 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15841 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15842 }
15843 return Register();
15844}
15845
15846Register fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
15847 if (RetVT.SimpleTy != MVT::v4i64)
15848 return Register();
15849 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15850 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15851 }
15852 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15853 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15854 }
15855 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15856 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
15857 }
15858 return Register();
15859}
15860
15861Register fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
15862 if (RetVT.SimpleTy != MVT::v8i64)
15863 return Register();
15864 if ((Subtarget->hasAVX512())) {
15865 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15866 }
15867 return Register();
15868}
15869
15870Register fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15871 switch (VT.SimpleTy) {
15872 case MVT::v16i8: return fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(RetVT, Op0, Op1);
15873 case MVT::v32i8: return fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(RetVT, Op0, Op1);
15874 case MVT::v64i8: return fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(RetVT, Op0, Op1);
15875 case MVT::v8i16: return fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(RetVT, Op0, Op1);
15876 case MVT::v16i16: return fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(RetVT, Op0, Op1);
15877 case MVT::v32i16: return fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(RetVT, Op0, Op1);
15878 case MVT::v4i32: return fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(RetVT, Op0, Op1);
15879 case MVT::v8i32: return fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(RetVT, Op0, Op1);
15880 case MVT::v16i32: return fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(RetVT, Op0, Op1);
15881 case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op1);
15882 case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op1);
15883 case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op1);
15884 default: return Register();
15885 }
15886}
15887
15888// FastEmit functions for X86ISD::BEXTR.
15889
15890Register fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15891 if (RetVT.SimpleTy != MVT::i32)
15892 return Register();
15893 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15894 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
15895 }
15896 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
15897 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr, RC: &X86::GR32RegClass, Op0, Op1);
15898 }
15899 return Register();
15900}
15901
15902Register fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15903 if (RetVT.SimpleTy != MVT::i64)
15904 return Register();
15905 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15906 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
15907 }
15908 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
15909 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr, RC: &X86::GR64RegClass, Op0, Op1);
15910 }
15911 return Register();
15912}
15913
15914Register fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15915 switch (VT.SimpleTy) {
15916 case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op1);
15917 case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op1);
15918 default: return Register();
15919 }
15920}
15921
15922// FastEmit functions for X86ISD::BZHI.
15923
15924Register fastEmit_X86ISD_BZHI_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15925 if (RetVT.SimpleTy != MVT::i32)
15926 return Register();
15927 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15928 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
15929 }
15930 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15931 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr, RC: &X86::GR32RegClass, Op0, Op1);
15932 }
15933 return Register();
15934}
15935
15936Register fastEmit_X86ISD_BZHI_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15937 if (RetVT.SimpleTy != MVT::i64)
15938 return Register();
15939 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15940 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
15941 }
15942 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15943 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr, RC: &X86::GR64RegClass, Op0, Op1);
15944 }
15945 return Register();
15946}
15947
15948Register fastEmit_X86ISD_BZHI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15949 switch (VT.SimpleTy) {
15950 case MVT::i32: return fastEmit_X86ISD_BZHI_MVT_i32_rr(RetVT, Op0, Op1);
15951 case MVT::i64: return fastEmit_X86ISD_BZHI_MVT_i64_rr(RetVT, Op0, Op1);
15952 default: return Register();
15953 }
15954}
15955
15956// FastEmit functions for X86ISD::FADDS.
15957
15958Register fastEmit_X86ISD_FADDS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15959 if (RetVT.SimpleTy != MVT::v8f16)
15960 return Register();
15961 if ((Subtarget->hasFP16())) {
15962 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15963 }
15964 return Register();
15965}
15966
15967Register fastEmit_X86ISD_FADDS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15968 if (RetVT.SimpleTy != MVT::v4f32)
15969 return Register();
15970 if ((Subtarget->hasAVX512())) {
15971 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15972 }
15973 return Register();
15974}
15975
15976Register fastEmit_X86ISD_FADDS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15977 if (RetVT.SimpleTy != MVT::v2f64)
15978 return Register();
15979 if ((Subtarget->hasAVX512())) {
15980 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15981 }
15982 return Register();
15983}
15984
15985Register fastEmit_X86ISD_FADDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15986 switch (VT.SimpleTy) {
15987 case MVT::v8f16: return fastEmit_X86ISD_FADDS_MVT_v8f16_rr(RetVT, Op0, Op1);
15988 case MVT::v4f32: return fastEmit_X86ISD_FADDS_MVT_v4f32_rr(RetVT, Op0, Op1);
15989 case MVT::v2f64: return fastEmit_X86ISD_FADDS_MVT_v2f64_rr(RetVT, Op0, Op1);
15990 default: return Register();
15991 }
15992}
15993
15994// FastEmit functions for X86ISD::FAND.
15995
15996Register fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
15997 if (RetVT.SimpleTy != MVT::f128)
15998 return Register();
15999 if ((Subtarget->hasVLX())) {
16000 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16001 }
16002 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16003 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16004 }
16005 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16006 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16007 }
16008 return Register();
16009}
16010
16011Register fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16012 if (RetVT.SimpleTy != MVT::v4f32)
16013 return Register();
16014 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16015}
16016
16017Register fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16018 switch (VT.SimpleTy) {
16019 case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op1);
16020 case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op1);
16021 default: return Register();
16022 }
16023}
16024
16025// FastEmit functions for X86ISD::FANDN.
16026
16027Register fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16028 if (RetVT.SimpleTy != MVT::v4f32)
16029 return Register();
16030 return fastEmitInst_rr(MachineInstOpcode: X86::ANDNPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16031}
16032
16033Register fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16034 switch (VT.SimpleTy) {
16035 case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op1);
16036 default: return Register();
16037 }
16038}
16039
16040// FastEmit functions for X86ISD::FDIVS.
16041
16042Register fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16043 if (RetVT.SimpleTy != MVT::v8f16)
16044 return Register();
16045 if ((Subtarget->hasFP16())) {
16046 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16047 }
16048 return Register();
16049}
16050
16051Register fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16052 if (RetVT.SimpleTy != MVT::v4f32)
16053 return Register();
16054 if ((Subtarget->hasAVX512())) {
16055 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16056 }
16057 return Register();
16058}
16059
16060Register fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16061 if (RetVT.SimpleTy != MVT::v2f64)
16062 return Register();
16063 if ((Subtarget->hasAVX512())) {
16064 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16065 }
16066 return Register();
16067}
16068
16069Register fastEmit_X86ISD_FDIVS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16070 switch (VT.SimpleTy) {
16071 case MVT::v8f16: return fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(RetVT, Op0, Op1);
16072 case MVT::v4f32: return fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(RetVT, Op0, Op1);
16073 case MVT::v2f64: return fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(RetVT, Op0, Op1);
16074 default: return Register();
16075 }
16076}
16077
16078// FastEmit functions for X86ISD::FGETEXPS.
16079
16080Register fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16081 if (RetVT.SimpleTy != MVT::v8f16)
16082 return Register();
16083 if ((Subtarget->hasFP16())) {
16084 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZr, RC: &X86::VR128XRegClass, Op0, Op1);
16085 }
16086 return Register();
16087}
16088
16089Register fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16090 if (RetVT.SimpleTy != MVT::v4f32)
16091 return Register();
16092 if ((Subtarget->hasAVX512())) {
16093 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZr, RC: &X86::VR128XRegClass, Op0, Op1);
16094 }
16095 return Register();
16096}
16097
16098Register fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16099 if (RetVT.SimpleTy != MVT::v2f64)
16100 return Register();
16101 if ((Subtarget->hasAVX512())) {
16102 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZr, RC: &X86::VR128XRegClass, Op0, Op1);
16103 }
16104 return Register();
16105}
16106
16107Register fastEmit_X86ISD_FGETEXPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16108 switch (VT.SimpleTy) {
16109 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(RetVT, Op0, Op1);
16110 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(RetVT, Op0, Op1);
16111 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(RetVT, Op0, Op1);
16112 default: return Register();
16113 }
16114}
16115
16116// FastEmit functions for X86ISD::FGETEXPS_SAE.
16117
16118Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16119 if (RetVT.SimpleTy != MVT::v8f16)
16120 return Register();
16121 if ((Subtarget->hasFP16())) {
16122 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZrb, RC: &X86::VR128XRegClass, Op0, Op1);
16123 }
16124 return Register();
16125}
16126
16127Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16128 if (RetVT.SimpleTy != MVT::v4f32)
16129 return Register();
16130 if ((Subtarget->hasAVX512())) {
16131 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZrb, RC: &X86::VR128XRegClass, Op0, Op1);
16132 }
16133 return Register();
16134}
16135
16136Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16137 if (RetVT.SimpleTy != MVT::v2f64)
16138 return Register();
16139 if ((Subtarget->hasAVX512())) {
16140 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZrb, RC: &X86::VR128XRegClass, Op0, Op1);
16141 }
16142 return Register();
16143}
16144
16145Register fastEmit_X86ISD_FGETEXPS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16146 switch (VT.SimpleTy) {
16147 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
16148 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16149 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16150 default: return Register();
16151 }
16152}
16153
16154// FastEmit functions for X86ISD::FHADD.
16155
16156Register fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16157 if (RetVT.SimpleTy != MVT::v4f32)
16158 return Register();
16159 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
16160 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16161 }
16162 if ((Subtarget->hasAVX())) {
16163 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16164 }
16165 return Register();
16166}
16167
16168Register fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16169 if (RetVT.SimpleTy != MVT::v8f32)
16170 return Register();
16171 if ((Subtarget->hasAVX())) {
16172 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16173 }
16174 return Register();
16175}
16176
16177Register fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16178 if (RetVT.SimpleTy != MVT::v2f64)
16179 return Register();
16180 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
16181 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16182 }
16183 if ((Subtarget->hasAVX())) {
16184 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16185 }
16186 return Register();
16187}
16188
16189Register fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16190 if (RetVT.SimpleTy != MVT::v4f64)
16191 return Register();
16192 if ((Subtarget->hasAVX())) {
16193 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16194 }
16195 return Register();
16196}
16197
16198Register fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16199 switch (VT.SimpleTy) {
16200 case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op1);
16201 case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op1);
16202 case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op1);
16203 case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op1);
16204 default: return Register();
16205 }
16206}
16207
16208// FastEmit functions for X86ISD::FHSUB.
16209
16210Register fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16211 if (RetVT.SimpleTy != MVT::v4f32)
16212 return Register();
16213 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
16214 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16215 }
16216 if ((Subtarget->hasAVX())) {
16217 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16218 }
16219 return Register();
16220}
16221
16222Register fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16223 if (RetVT.SimpleTy != MVT::v8f32)
16224 return Register();
16225 if ((Subtarget->hasAVX())) {
16226 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16227 }
16228 return Register();
16229}
16230
16231Register fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16232 if (RetVT.SimpleTy != MVT::v2f64)
16233 return Register();
16234 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
16235 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16236 }
16237 if ((Subtarget->hasAVX())) {
16238 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16239 }
16240 return Register();
16241}
16242
16243Register fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16244 if (RetVT.SimpleTy != MVT::v4f64)
16245 return Register();
16246 if ((Subtarget->hasAVX())) {
16247 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16248 }
16249 return Register();
16250}
16251
16252Register fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16253 switch (VT.SimpleTy) {
16254 case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
16255 case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
16256 case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
16257 case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
16258 default: return Register();
16259 }
16260}
16261
16262// FastEmit functions for X86ISD::FMAXC.
16263
16264Register fastEmit_X86ISD_FMAXC_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16265 if (RetVT.SimpleTy != MVT::f16)
16266 return Register();
16267 if ((Subtarget->hasAVX512())) {
16268 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16269 }
16270 return Register();
16271}
16272
16273Register fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16274 if (RetVT.SimpleTy != MVT::f32)
16275 return Register();
16276 if ((Subtarget->hasAVX512())) {
16277 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16278 }
16279 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16280 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16281 }
16282 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16283 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16284 }
16285 return Register();
16286}
16287
16288Register fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16289 if (RetVT.SimpleTy != MVT::f64)
16290 return Register();
16291 if ((Subtarget->hasAVX512())) {
16292 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16293 }
16294 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16295 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16296 }
16297 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16298 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16299 }
16300 return Register();
16301}
16302
16303Register fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16304 if (RetVT.SimpleTy != MVT::v8f16)
16305 return Register();
16306 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16307 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16308 }
16309 return Register();
16310}
16311
16312Register fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
16313 if (RetVT.SimpleTy != MVT::v16f16)
16314 return Register();
16315 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16316 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16317 }
16318 return Register();
16319}
16320
16321Register fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16322 if (RetVT.SimpleTy != MVT::v32f16)
16323 return Register();
16324 if ((Subtarget->hasFP16())) {
16325 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16326 }
16327 return Register();
16328}
16329
16330Register fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16331 if (RetVT.SimpleTy != MVT::v4f32)
16332 return Register();
16333 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16334 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16335 }
16336 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16337 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16338 }
16339 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16340 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16341 }
16342 return Register();
16343}
16344
16345Register fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16346 if (RetVT.SimpleTy != MVT::v8f32)
16347 return Register();
16348 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16349 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16350 }
16351 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16352 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16353 }
16354 return Register();
16355}
16356
16357Register fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16358 if (RetVT.SimpleTy != MVT::v16f32)
16359 return Register();
16360 if ((Subtarget->hasAVX512())) {
16361 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16362 }
16363 return Register();
16364}
16365
16366Register fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16367 if (RetVT.SimpleTy != MVT::v2f64)
16368 return Register();
16369 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16370 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16371 }
16372 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16373 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16374 }
16375 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16376 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16377 }
16378 return Register();
16379}
16380
16381Register fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16382 if (RetVT.SimpleTy != MVT::v4f64)
16383 return Register();
16384 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16385 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16386 }
16387 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16388 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16389 }
16390 return Register();
16391}
16392
16393Register fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16394 if (RetVT.SimpleTy != MVT::v8f64)
16395 return Register();
16396 if ((Subtarget->hasAVX512())) {
16397 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16398 }
16399 return Register();
16400}
16401
16402Register fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16403 switch (VT.SimpleTy) {
16404 case MVT::f16: return fastEmit_X86ISD_FMAXC_MVT_f16_rr(RetVT, Op0, Op1);
16405 case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op1);
16406 case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op1);
16407 case MVT::v8f16: return fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(RetVT, Op0, Op1);
16408 case MVT::v16f16: return fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(RetVT, Op0, Op1);
16409 case MVT::v32f16: return fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(RetVT, Op0, Op1);
16410 case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op1);
16411 case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op1);
16412 case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op1);
16413 case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op1);
16414 case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op1);
16415 case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op1);
16416 default: return Register();
16417 }
16418}
16419
16420// FastEmit functions for X86ISD::FMAXS.
16421
16422Register fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16423 if (RetVT.SimpleTy != MVT::v8f16)
16424 return Register();
16425 if ((Subtarget->hasFP16())) {
16426 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16427 }
16428 return Register();
16429}
16430
16431Register fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16432 if (RetVT.SimpleTy != MVT::v4f32)
16433 return Register();
16434 if ((Subtarget->hasAVX512())) {
16435 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16436 }
16437 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16438 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16439 }
16440 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16441 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16442 }
16443 return Register();
16444}
16445
16446Register fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16447 if (RetVT.SimpleTy != MVT::v2f64)
16448 return Register();
16449 if ((Subtarget->hasAVX512())) {
16450 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16451 }
16452 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16453 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16454 }
16455 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16456 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16457 }
16458 return Register();
16459}
16460
16461Register fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16462 switch (VT.SimpleTy) {
16463 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(RetVT, Op0, Op1);
16464 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op1);
16465 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op1);
16466 default: return Register();
16467 }
16468}
16469
16470// FastEmit functions for X86ISD::FMAXS_SAE.
16471
16472Register fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16473 if (RetVT.SimpleTy != MVT::v8f16)
16474 return Register();
16475 if ((Subtarget->hasFP16())) {
16476 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16477 }
16478 return Register();
16479}
16480
16481Register fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16482 if (RetVT.SimpleTy != MVT::v4f32)
16483 return Register();
16484 if ((Subtarget->hasAVX512())) {
16485 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16486 }
16487 return Register();
16488}
16489
16490Register fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16491 if (RetVT.SimpleTy != MVT::v2f64)
16492 return Register();
16493 if ((Subtarget->hasAVX512())) {
16494 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16495 }
16496 return Register();
16497}
16498
16499Register fastEmit_X86ISD_FMAXS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16500 switch (VT.SimpleTy) {
16501 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
16502 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16503 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16504 default: return Register();
16505 }
16506}
16507
16508// FastEmit functions for X86ISD::FMAX_SAE.
16509
16510Register fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16511 if (RetVT.SimpleTy != MVT::v32f16)
16512 return Register();
16513 if ((Subtarget->hasFP16())) {
16514 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16515 }
16516 return Register();
16517}
16518
16519Register fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16520 if (RetVT.SimpleTy != MVT::v16f32)
16521 return Register();
16522 if ((Subtarget->hasAVX512())) {
16523 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16524 }
16525 return Register();
16526}
16527
16528Register fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16529 if (RetVT.SimpleTy != MVT::v8f64)
16530 return Register();
16531 if ((Subtarget->hasAVX512())) {
16532 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16533 }
16534 return Register();
16535}
16536
16537Register fastEmit_X86ISD_FMAX_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16538 switch (VT.SimpleTy) {
16539 case MVT::v32f16: return fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
16540 case MVT::v16f32: return fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
16541 case MVT::v8f64: return fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
16542 default: return Register();
16543 }
16544}
16545
16546// FastEmit functions for X86ISD::FMINC.
16547
16548Register fastEmit_X86ISD_FMINC_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16549 if (RetVT.SimpleTy != MVT::f16)
16550 return Register();
16551 if ((Subtarget->hasAVX512())) {
16552 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16553 }
16554 return Register();
16555}
16556
16557Register fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16558 if (RetVT.SimpleTy != MVT::f32)
16559 return Register();
16560 if ((Subtarget->hasAVX512())) {
16561 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16562 }
16563 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16564 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16565 }
16566 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16567 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16568 }
16569 return Register();
16570}
16571
16572Register fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16573 if (RetVT.SimpleTy != MVT::f64)
16574 return Register();
16575 if ((Subtarget->hasAVX512())) {
16576 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16577 }
16578 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16579 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16580 }
16581 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16582 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16583 }
16584 return Register();
16585}
16586
16587Register fastEmit_X86ISD_FMINC_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16588 if (RetVT.SimpleTy != MVT::v8f16)
16589 return Register();
16590 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16591 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16592 }
16593 return Register();
16594}
16595
16596Register fastEmit_X86ISD_FMINC_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
16597 if (RetVT.SimpleTy != MVT::v16f16)
16598 return Register();
16599 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16600 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16601 }
16602 return Register();
16603}
16604
16605Register fastEmit_X86ISD_FMINC_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16606 if (RetVT.SimpleTy != MVT::v32f16)
16607 return Register();
16608 if ((Subtarget->hasFP16())) {
16609 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16610 }
16611 return Register();
16612}
16613
16614Register fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16615 if (RetVT.SimpleTy != MVT::v4f32)
16616 return Register();
16617 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16618 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16619 }
16620 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16621 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16622 }
16623 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16624 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16625 }
16626 return Register();
16627}
16628
16629Register fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16630 if (RetVT.SimpleTy != MVT::v8f32)
16631 return Register();
16632 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16633 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16634 }
16635 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16636 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16637 }
16638 return Register();
16639}
16640
16641Register fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16642 if (RetVT.SimpleTy != MVT::v16f32)
16643 return Register();
16644 if ((Subtarget->hasAVX512())) {
16645 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16646 }
16647 return Register();
16648}
16649
16650Register fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16651 if (RetVT.SimpleTy != MVT::v2f64)
16652 return Register();
16653 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16654 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16655 }
16656 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16657 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16658 }
16659 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16660 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16661 }
16662 return Register();
16663}
16664
16665Register fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16666 if (RetVT.SimpleTy != MVT::v4f64)
16667 return Register();
16668 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16669 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16670 }
16671 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16672 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16673 }
16674 return Register();
16675}
16676
16677Register fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16678 if (RetVT.SimpleTy != MVT::v8f64)
16679 return Register();
16680 if ((Subtarget->hasAVX512())) {
16681 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16682 }
16683 return Register();
16684}
16685
16686Register fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16687 switch (VT.SimpleTy) {
16688 case MVT::f16: return fastEmit_X86ISD_FMINC_MVT_f16_rr(RetVT, Op0, Op1);
16689 case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op1);
16690 case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op1);
16691 case MVT::v8f16: return fastEmit_X86ISD_FMINC_MVT_v8f16_rr(RetVT, Op0, Op1);
16692 case MVT::v16f16: return fastEmit_X86ISD_FMINC_MVT_v16f16_rr(RetVT, Op0, Op1);
16693 case MVT::v32f16: return fastEmit_X86ISD_FMINC_MVT_v32f16_rr(RetVT, Op0, Op1);
16694 case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op1);
16695 case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op1);
16696 case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op1);
16697 case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op1);
16698 case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op1);
16699 case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op1);
16700 default: return Register();
16701 }
16702}
16703
16704// FastEmit functions for X86ISD::FMINS.
16705
16706Register fastEmit_X86ISD_FMINS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16707 if (RetVT.SimpleTy != MVT::v8f16)
16708 return Register();
16709 if ((Subtarget->hasFP16())) {
16710 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16711 }
16712 return Register();
16713}
16714
16715Register fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16716 if (RetVT.SimpleTy != MVT::v4f32)
16717 return Register();
16718 if ((Subtarget->hasAVX512())) {
16719 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16720 }
16721 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16722 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16723 }
16724 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16725 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16726 }
16727 return Register();
16728}
16729
16730Register fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16731 if (RetVT.SimpleTy != MVT::v2f64)
16732 return Register();
16733 if ((Subtarget->hasAVX512())) {
16734 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16735 }
16736 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16737 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16738 }
16739 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16740 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16741 }
16742 return Register();
16743}
16744
16745Register fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16746 switch (VT.SimpleTy) {
16747 case MVT::v8f16: return fastEmit_X86ISD_FMINS_MVT_v8f16_rr(RetVT, Op0, Op1);
16748 case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op1);
16749 case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op1);
16750 default: return Register();
16751 }
16752}
16753
16754// FastEmit functions for X86ISD::FMINS_SAE.
16755
16756Register fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16757 if (RetVT.SimpleTy != MVT::v8f16)
16758 return Register();
16759 if ((Subtarget->hasFP16())) {
16760 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16761 }
16762 return Register();
16763}
16764
16765Register fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16766 if (RetVT.SimpleTy != MVT::v4f32)
16767 return Register();
16768 if ((Subtarget->hasAVX512())) {
16769 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16770 }
16771 return Register();
16772}
16773
16774Register fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16775 if (RetVT.SimpleTy != MVT::v2f64)
16776 return Register();
16777 if ((Subtarget->hasAVX512())) {
16778 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16779 }
16780 return Register();
16781}
16782
16783Register fastEmit_X86ISD_FMINS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16784 switch (VT.SimpleTy) {
16785 case MVT::v8f16: return fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
16786 case MVT::v4f32: return fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16787 case MVT::v2f64: return fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16788 default: return Register();
16789 }
16790}
16791
16792// FastEmit functions for X86ISD::FMIN_SAE.
16793
16794Register fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16795 if (RetVT.SimpleTy != MVT::v32f16)
16796 return Register();
16797 if ((Subtarget->hasFP16())) {
16798 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16799 }
16800 return Register();
16801}
16802
16803Register fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16804 if (RetVT.SimpleTy != MVT::v16f32)
16805 return Register();
16806 if ((Subtarget->hasAVX512())) {
16807 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16808 }
16809 return Register();
16810}
16811
16812Register fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16813 if (RetVT.SimpleTy != MVT::v8f64)
16814 return Register();
16815 if ((Subtarget->hasAVX512())) {
16816 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16817 }
16818 return Register();
16819}
16820
16821Register fastEmit_X86ISD_FMIN_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16822 switch (VT.SimpleTy) {
16823 case MVT::v32f16: return fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
16824 case MVT::v16f32: return fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
16825 case MVT::v8f64: return fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
16826 default: return Register();
16827 }
16828}
16829
16830// FastEmit functions for X86ISD::FMULS.
16831
16832Register fastEmit_X86ISD_FMULS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16833 if (RetVT.SimpleTy != MVT::v8f16)
16834 return Register();
16835 if ((Subtarget->hasFP16())) {
16836 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16837 }
16838 return Register();
16839}
16840
16841Register fastEmit_X86ISD_FMULS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16842 if (RetVT.SimpleTy != MVT::v4f32)
16843 return Register();
16844 if ((Subtarget->hasAVX512())) {
16845 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16846 }
16847 return Register();
16848}
16849
16850Register fastEmit_X86ISD_FMULS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16851 if (RetVT.SimpleTy != MVT::v2f64)
16852 return Register();
16853 if ((Subtarget->hasAVX512())) {
16854 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16855 }
16856 return Register();
16857}
16858
16859Register fastEmit_X86ISD_FMULS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16860 switch (VT.SimpleTy) {
16861 case MVT::v8f16: return fastEmit_X86ISD_FMULS_MVT_v8f16_rr(RetVT, Op0, Op1);
16862 case MVT::v4f32: return fastEmit_X86ISD_FMULS_MVT_v4f32_rr(RetVT, Op0, Op1);
16863 case MVT::v2f64: return fastEmit_X86ISD_FMULS_MVT_v2f64_rr(RetVT, Op0, Op1);
16864 default: return Register();
16865 }
16866}
16867
16868// FastEmit functions for X86ISD::FOR.
16869
16870Register fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
16871 if (RetVT.SimpleTy != MVT::f128)
16872 return Register();
16873 if ((Subtarget->hasVLX())) {
16874 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16875 }
16876 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16877 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16878 }
16879 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16880 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16881 }
16882 return Register();
16883}
16884
16885Register fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16886 if (RetVT.SimpleTy != MVT::v4f32)
16887 return Register();
16888 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16889}
16890
16891Register fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16892 switch (VT.SimpleTy) {
16893 case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op1);
16894 case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op1);
16895 default: return Register();
16896 }
16897}
16898
16899// FastEmit functions for X86ISD::FP80_ADD.
16900
16901Register fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
16902 if (RetVT.SimpleTy != MVT::f80)
16903 return Register();
16904 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
16905}
16906
16907Register fastEmit_X86ISD_FP80_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16908 switch (VT.SimpleTy) {
16909 case MVT::f80: return fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
16910 default: return Register();
16911 }
16912}
16913
16914// FastEmit functions for X86ISD::FSQRTS.
16915
16916Register fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16917 if (RetVT.SimpleTy != MVT::v8f16)
16918 return Register();
16919 if ((Subtarget->hasFP16())) {
16920 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSHZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16921 }
16922 return Register();
16923}
16924
16925Register fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16926 if (RetVT.SimpleTy != MVT::v4f32)
16927 return Register();
16928 if ((Subtarget->hasAVX512())) {
16929 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSSZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16930 }
16931 return Register();
16932}
16933
16934Register fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16935 if (RetVT.SimpleTy != MVT::v2f64)
16936 return Register();
16937 if ((Subtarget->hasAVX512())) {
16938 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSDZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16939 }
16940 return Register();
16941}
16942
16943Register fastEmit_X86ISD_FSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16944 switch (VT.SimpleTy) {
16945 case MVT::v8f16: return fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(RetVT, Op0, Op1);
16946 case MVT::v4f32: return fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
16947 case MVT::v2f64: return fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
16948 default: return Register();
16949 }
16950}
16951
16952// FastEmit functions for X86ISD::FSUBS.
16953
16954Register fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16955 if (RetVT.SimpleTy != MVT::v8f16)
16956 return Register();
16957 if ((Subtarget->hasFP16())) {
16958 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16959 }
16960 return Register();
16961}
16962
16963Register fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16964 if (RetVT.SimpleTy != MVT::v4f32)
16965 return Register();
16966 if ((Subtarget->hasAVX512())) {
16967 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16968 }
16969 return Register();
16970}
16971
16972Register fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16973 if (RetVT.SimpleTy != MVT::v2f64)
16974 return Register();
16975 if ((Subtarget->hasAVX512())) {
16976 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16977 }
16978 return Register();
16979}
16980
16981Register fastEmit_X86ISD_FSUBS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16982 switch (VT.SimpleTy) {
16983 case MVT::v8f16: return fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(RetVT, Op0, Op1);
16984 case MVT::v4f32: return fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(RetVT, Op0, Op1);
16985 case MVT::v2f64: return fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(RetVT, Op0, Op1);
16986 default: return Register();
16987 }
16988}
16989
16990// FastEmit functions for X86ISD::FXOR.
16991
16992Register fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
16993 if (RetVT.SimpleTy != MVT::f128)
16994 return Register();
16995 if ((Subtarget->hasVLX())) {
16996 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16997 }
16998 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16999 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17000 }
17001 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17002 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17003 }
17004 return Register();
17005}
17006
17007Register fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17008 if (RetVT.SimpleTy != MVT::v4f32)
17009 return Register();
17010 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17011}
17012
17013Register fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17014 switch (VT.SimpleTy) {
17015 case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op1);
17016 case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op1);
17017 default: return Register();
17018 }
17019}
17020
17021// FastEmit functions for X86ISD::GF2P8MULB.
17022
17023Register fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17024 if (RetVT.SimpleTy != MVT::v16i8)
17025 return Register();
17026 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
17027 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17028 }
17029 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
17030 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
17031 }
17032 if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17033 return fastEmitInst_rr(MachineInstOpcode: X86::GF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
17034 }
17035 return Register();
17036}
17037
17038Register fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17039 if (RetVT.SimpleTy != MVT::v32i8)
17040 return Register();
17041 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
17042 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17043 }
17044 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
17045 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17046 }
17047 return Register();
17048}
17049
17050Register fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17051 if (RetVT.SimpleTy != MVT::v64i8)
17052 return Register();
17053 if ((Subtarget->hasAVX512()) && (Subtarget->hasGFNI())) {
17054 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17055 }
17056 return Register();
17057}
17058
17059Register fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17060 switch (VT.SimpleTy) {
17061 case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op1);
17062 case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op1);
17063 case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op1);
17064 default: return Register();
17065 }
17066}
17067
17068// FastEmit functions for X86ISD::HADD.
17069
17070Register fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17071 if (RetVT.SimpleTy != MVT::v8i16)
17072 return Register();
17073 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17074 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17075 }
17076 if ((Subtarget->hasAVX())) {
17077 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17078 }
17079 return Register();
17080}
17081
17082Register fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17083 if (RetVT.SimpleTy != MVT::v16i16)
17084 return Register();
17085 if ((Subtarget->hasAVX2())) {
17086 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17087 }
17088 return Register();
17089}
17090
17091Register fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17092 if (RetVT.SimpleTy != MVT::v4i32)
17093 return Register();
17094 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17095 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
17096 }
17097 if ((Subtarget->hasAVX())) {
17098 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
17099 }
17100 return Register();
17101}
17102
17103Register fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17104 if (RetVT.SimpleTy != MVT::v8i32)
17105 return Register();
17106 if ((Subtarget->hasAVX2())) {
17107 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17108 }
17109 return Register();
17110}
17111
17112Register fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17113 switch (VT.SimpleTy) {
17114 case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op1);
17115 case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op1);
17116 case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op1);
17117 case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op1);
17118 default: return Register();
17119 }
17120}
17121
17122// FastEmit functions for X86ISD::HADDS.
17123
17124Register fastEmit_X86ISD_HADDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17125 if (RetVT.SimpleTy != MVT::v8i16)
17126 return Register();
17127 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17128 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17129 }
17130 if ((Subtarget->hasAVX())) {
17131 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17132 }
17133 return Register();
17134}
17135
17136Register fastEmit_X86ISD_HADDS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17137 if (RetVT.SimpleTy != MVT::v16i16)
17138 return Register();
17139 if ((Subtarget->hasAVX2())) {
17140 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17141 }
17142 return Register();
17143}
17144
17145Register fastEmit_X86ISD_HADDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17146 switch (VT.SimpleTy) {
17147 case MVT::v8i16: return fastEmit_X86ISD_HADDS_MVT_v8i16_rr(RetVT, Op0, Op1);
17148 case MVT::v16i16: return fastEmit_X86ISD_HADDS_MVT_v16i16_rr(RetVT, Op0, Op1);
17149 default: return Register();
17150 }
17151}
17152
17153// FastEmit functions for X86ISD::HSUB.
17154
17155Register fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17156 if (RetVT.SimpleTy != MVT::v8i16)
17157 return Register();
17158 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17159 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17160 }
17161 if ((Subtarget->hasAVX())) {
17162 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17163 }
17164 return Register();
17165}
17166
17167Register fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17168 if (RetVT.SimpleTy != MVT::v16i16)
17169 return Register();
17170 if ((Subtarget->hasAVX2())) {
17171 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17172 }
17173 return Register();
17174}
17175
17176Register fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17177 if (RetVT.SimpleTy != MVT::v4i32)
17178 return Register();
17179 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17180 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
17181 }
17182 if ((Subtarget->hasAVX())) {
17183 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
17184 }
17185 return Register();
17186}
17187
17188Register fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17189 if (RetVT.SimpleTy != MVT::v8i32)
17190 return Register();
17191 if ((Subtarget->hasAVX2())) {
17192 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17193 }
17194 return Register();
17195}
17196
17197Register fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17198 switch (VT.SimpleTy) {
17199 case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op1);
17200 case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op1);
17201 case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op1);
17202 case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op1);
17203 default: return Register();
17204 }
17205}
17206
17207// FastEmit functions for X86ISD::HSUBS.
17208
17209Register fastEmit_X86ISD_HSUBS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17210 if (RetVT.SimpleTy != MVT::v8i16)
17211 return Register();
17212 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17213 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17214 }
17215 if ((Subtarget->hasAVX())) {
17216 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17217 }
17218 return Register();
17219}
17220
17221Register fastEmit_X86ISD_HSUBS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17222 if (RetVT.SimpleTy != MVT::v16i16)
17223 return Register();
17224 if ((Subtarget->hasAVX2())) {
17225 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17226 }
17227 return Register();
17228}
17229
17230Register fastEmit_X86ISD_HSUBS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17231 switch (VT.SimpleTy) {
17232 case MVT::v8i16: return fastEmit_X86ISD_HSUBS_MVT_v8i16_rr(RetVT, Op0, Op1);
17233 case MVT::v16i16: return fastEmit_X86ISD_HSUBS_MVT_v16i16_rr(RetVT, Op0, Op1);
17234 default: return Register();
17235 }
17236}
17237
17238// FastEmit functions for X86ISD::KADD.
17239
17240Register fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
17241 if (RetVT.SimpleTy != MVT::v8i1)
17242 return Register();
17243 if ((Subtarget->hasDQI())) {
17244 return fastEmitInst_rr(MachineInstOpcode: X86::KADDBkk, RC: &X86::VK8RegClass, Op0, Op1);
17245 }
17246 return Register();
17247}
17248
17249Register fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
17250 if (RetVT.SimpleTy != MVT::v16i1)
17251 return Register();
17252 if ((Subtarget->hasDQI())) {
17253 return fastEmitInst_rr(MachineInstOpcode: X86::KADDWkk, RC: &X86::VK16RegClass, Op0, Op1);
17254 }
17255 return Register();
17256}
17257
17258Register fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
17259 if (RetVT.SimpleTy != MVT::v32i1)
17260 return Register();
17261 if ((Subtarget->hasBWI())) {
17262 return fastEmitInst_rr(MachineInstOpcode: X86::KADDDkk, RC: &X86::VK32RegClass, Op0, Op1);
17263 }
17264 return Register();
17265}
17266
17267Register fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
17268 if (RetVT.SimpleTy != MVT::v64i1)
17269 return Register();
17270 if ((Subtarget->hasBWI())) {
17271 return fastEmitInst_rr(MachineInstOpcode: X86::KADDQkk, RC: &X86::VK64RegClass, Op0, Op1);
17272 }
17273 return Register();
17274}
17275
17276Register fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17277 switch (VT.SimpleTy) {
17278 case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op1);
17279 case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op1);
17280 case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op1);
17281 case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op1);
17282 default: return Register();
17283 }
17284}
17285
17286// FastEmit functions for X86ISD::MOVHLPS.
17287
17288Register fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17289 if (RetVT.SimpleTy != MVT::v4f32)
17290 return Register();
17291 if ((Subtarget->hasAVX512())) {
17292 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17293 }
17294 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17295 return fastEmitInst_rr(MachineInstOpcode: X86::MOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17296 }
17297 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17298 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17299 }
17300 return Register();
17301}
17302
17303Register fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17304 switch (VT.SimpleTy) {
17305 case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op1);
17306 default: return Register();
17307 }
17308}
17309
17310// FastEmit functions for X86ISD::MOVLHPS.
17311
17312Register fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17313 if (RetVT.SimpleTy != MVT::v4f32)
17314 return Register();
17315 if ((Subtarget->hasAVX512())) {
17316 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17317 }
17318 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17319 return fastEmitInst_rr(MachineInstOpcode: X86::MOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17320 }
17321 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17322 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17323 }
17324 return Register();
17325}
17326
17327Register fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17328 switch (VT.SimpleTy) {
17329 case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op1);
17330 default: return Register();
17331 }
17332}
17333
17334// FastEmit functions for X86ISD::MOVSD.
17335
17336Register fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17337 if (RetVT.SimpleTy != MVT::v2f64)
17338 return Register();
17339 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
17340 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17341 }
17342 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17343 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
17344 }
17345 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17346 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
17347 }
17348 return Register();
17349}
17350
17351Register fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17352 switch (VT.SimpleTy) {
17353 case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op1);
17354 default: return Register();
17355 }
17356}
17357
17358// FastEmit functions for X86ISD::MOVSH.
17359
17360Register fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17361 if (RetVT.SimpleTy != MVT::v8f16)
17362 return Register();
17363 if ((Subtarget->hasFP16())) {
17364 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17365 }
17366 return Register();
17367}
17368
17369Register fastEmit_X86ISD_MOVSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17370 switch (VT.SimpleTy) {
17371 case MVT::v8f16: return fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(RetVT, Op0, Op1);
17372 default: return Register();
17373 }
17374}
17375
17376// FastEmit functions for X86ISD::MOVSS.
17377
17378Register fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17379 if (RetVT.SimpleTy != MVT::v4f32)
17380 return Register();
17381 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
17382 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17383 }
17384 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17385 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
17386 }
17387 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17388 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
17389 }
17390 return Register();
17391}
17392
17393Register fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17394 switch (VT.SimpleTy) {
17395 case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op1);
17396 default: return Register();
17397 }
17398}
17399
17400// FastEmit functions for X86ISD::MULHRS.
17401
17402Register fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17403 if (RetVT.SimpleTy != MVT::v8i16)
17404 return Register();
17405 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17406 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17407 }
17408 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17409 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17410 }
17411 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17412 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17413 }
17414 return Register();
17415}
17416
17417Register fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17418 if (RetVT.SimpleTy != MVT::v16i16)
17419 return Register();
17420 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17421 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17422 }
17423 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17424 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17425 }
17426 return Register();
17427}
17428
17429Register fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17430 if (RetVT.SimpleTy != MVT::v32i16)
17431 return Register();
17432 if ((Subtarget->hasBWI())) {
17433 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17434 }
17435 return Register();
17436}
17437
17438Register fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17439 switch (VT.SimpleTy) {
17440 case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op1);
17441 case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op1);
17442 case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op1);
17443 default: return Register();
17444 }
17445}
17446
17447// FastEmit functions for X86ISD::MULTISHIFT.
17448
17449Register fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17450 if (RetVT.SimpleTy != MVT::v16i8)
17451 return Register();
17452 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17453 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17454 }
17455 return Register();
17456}
17457
17458Register fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17459 if (RetVT.SimpleTy != MVT::v32i8)
17460 return Register();
17461 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17462 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17463 }
17464 return Register();
17465}
17466
17467Register fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17468 if (RetVT.SimpleTy != MVT::v64i8)
17469 return Register();
17470 if ((Subtarget->hasVBMI())) {
17471 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17472 }
17473 return Register();
17474}
17475
17476Register fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17477 switch (VT.SimpleTy) {
17478 case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op1);
17479 case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op1);
17480 case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op1);
17481 default: return Register();
17482 }
17483}
17484
17485// FastEmit functions for X86ISD::PACKSS.
17486
17487Register fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17488 if (RetVT.SimpleTy != MVT::v16i8)
17489 return Register();
17490 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17491 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17492 }
17493 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17494 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17495 }
17496 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17497 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17498 }
17499 return Register();
17500}
17501
17502Register fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17503 if (RetVT.SimpleTy != MVT::v32i8)
17504 return Register();
17505 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17506 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17507 }
17508 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17509 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17510 }
17511 return Register();
17512}
17513
17514Register fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17515 if (RetVT.SimpleTy != MVT::v64i8)
17516 return Register();
17517 if ((Subtarget->hasBWI())) {
17518 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17519 }
17520 return Register();
17521}
17522
17523Register fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17524 if (RetVT.SimpleTy != MVT::v8i16)
17525 return Register();
17526 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17527 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17528 }
17529 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17530 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17531 }
17532 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17533 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17534 }
17535 return Register();
17536}
17537
17538Register fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17539 if (RetVT.SimpleTy != MVT::v16i16)
17540 return Register();
17541 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17542 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17543 }
17544 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17545 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17546 }
17547 return Register();
17548}
17549
17550Register fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
17551 if (RetVT.SimpleTy != MVT::v32i16)
17552 return Register();
17553 if ((Subtarget->hasBWI())) {
17554 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17555 }
17556 return Register();
17557}
17558
17559Register fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17560 switch (VT.SimpleTy) {
17561 case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op1);
17562 case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op1);
17563 case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op1);
17564 case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op1);
17565 case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op1);
17566 case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op1);
17567 default: return Register();
17568 }
17569}
17570
17571// FastEmit functions for X86ISD::PACKUS.
17572
17573Register fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17574 if (RetVT.SimpleTy != MVT::v16i8)
17575 return Register();
17576 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17577 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17578 }
17579 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17580 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17581 }
17582 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17583 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17584 }
17585 return Register();
17586}
17587
17588Register fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17589 if (RetVT.SimpleTy != MVT::v32i8)
17590 return Register();
17591 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17592 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17593 }
17594 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17595 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17596 }
17597 return Register();
17598}
17599
17600Register fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17601 if (RetVT.SimpleTy != MVT::v64i8)
17602 return Register();
17603 if ((Subtarget->hasBWI())) {
17604 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17605 }
17606 return Register();
17607}
17608
17609Register fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17610 if (RetVT.SimpleTy != MVT::v8i16)
17611 return Register();
17612 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17613 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17614 }
17615 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17616 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17617 }
17618 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17619 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17620 }
17621 return Register();
17622}
17623
17624Register fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17625 if (RetVT.SimpleTy != MVT::v16i16)
17626 return Register();
17627 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17628 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17629 }
17630 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17631 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17632 }
17633 return Register();
17634}
17635
17636Register fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
17637 if (RetVT.SimpleTy != MVT::v32i16)
17638 return Register();
17639 if ((Subtarget->hasBWI())) {
17640 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17641 }
17642 return Register();
17643}
17644
17645Register fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17646 switch (VT.SimpleTy) {
17647 case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op1);
17648 case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op1);
17649 case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op1);
17650 case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op1);
17651 case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op1);
17652 case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op1);
17653 default: return Register();
17654 }
17655}
17656
17657// FastEmit functions for X86ISD::PCMPEQ.
17658
17659Register fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17660 if (RetVT.SimpleTy != MVT::v16i8)
17661 return Register();
17662 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17663 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
17664 }
17665 if ((Subtarget->hasAVX()) && (true)) {
17666 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
17667 }
17668 return Register();
17669}
17670
17671Register fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17672 if (RetVT.SimpleTy != MVT::v32i8)
17673 return Register();
17674 if ((Subtarget->hasAVX2()) && (true)) {
17675 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17676 }
17677 return Register();
17678}
17679
17680Register fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17681 if (RetVT.SimpleTy != MVT::v8i16)
17682 return Register();
17683 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17684 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
17685 }
17686 if ((Subtarget->hasAVX()) && (true)) {
17687 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
17688 }
17689 return Register();
17690}
17691
17692Register fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17693 if (RetVT.SimpleTy != MVT::v16i16)
17694 return Register();
17695 if ((Subtarget->hasAVX2()) && (true)) {
17696 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17697 }
17698 return Register();
17699}
17700
17701Register fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17702 if (RetVT.SimpleTy != MVT::v4i32)
17703 return Register();
17704 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17705 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
17706 }
17707 if ((Subtarget->hasAVX()) && (true)) {
17708 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
17709 }
17710 return Register();
17711}
17712
17713Register fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17714 if (RetVT.SimpleTy != MVT::v8i32)
17715 return Register();
17716 if ((Subtarget->hasAVX2()) && (true)) {
17717 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17718 }
17719 return Register();
17720}
17721
17722Register fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17723 if (RetVT.SimpleTy != MVT::v2i64)
17724 return Register();
17725 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17726 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
17727 }
17728 if ((Subtarget->hasAVX())) {
17729 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
17730 }
17731 return Register();
17732}
17733
17734Register fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17735 if (RetVT.SimpleTy != MVT::v4i64)
17736 return Register();
17737 if ((Subtarget->hasAVX2())) {
17738 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17739 }
17740 return Register();
17741}
17742
17743Register fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17744 switch (VT.SimpleTy) {
17745 case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op1);
17746 case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op1);
17747 case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op1);
17748 case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op1);
17749 case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op1);
17750 case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op1);
17751 case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17752 case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17753 default: return Register();
17754 }
17755}
17756
17757// FastEmit functions for X86ISD::PCMPGT.
17758
17759Register fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17760 if (RetVT.SimpleTy != MVT::v16i8)
17761 return Register();
17762 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17763 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
17764 }
17765 if ((Subtarget->hasAVX()) && (true)) {
17766 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
17767 }
17768 return Register();
17769}
17770
17771Register fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17772 if (RetVT.SimpleTy != MVT::v32i8)
17773 return Register();
17774 if ((Subtarget->hasAVX2()) && (true)) {
17775 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17776 }
17777 return Register();
17778}
17779
17780Register fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17781 if (RetVT.SimpleTy != MVT::v8i16)
17782 return Register();
17783 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17784 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
17785 }
17786 if ((Subtarget->hasAVX()) && (true)) {
17787 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
17788 }
17789 return Register();
17790}
17791
17792Register fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17793 if (RetVT.SimpleTy != MVT::v16i16)
17794 return Register();
17795 if ((Subtarget->hasAVX2()) && (true)) {
17796 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17797 }
17798 return Register();
17799}
17800
17801Register fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17802 if (RetVT.SimpleTy != MVT::v4i32)
17803 return Register();
17804 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17805 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
17806 }
17807 if ((Subtarget->hasAVX()) && (true)) {
17808 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
17809 }
17810 return Register();
17811}
17812
17813Register fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17814 if (RetVT.SimpleTy != MVT::v8i32)
17815 return Register();
17816 if ((Subtarget->hasAVX2()) && (true)) {
17817 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17818 }
17819 return Register();
17820}
17821
17822Register fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17823 if (RetVT.SimpleTy != MVT::v2i64)
17824 return Register();
17825 if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) {
17826 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
17827 }
17828 if ((Subtarget->hasAVX())) {
17829 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
17830 }
17831 return Register();
17832}
17833
17834Register fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17835 if (RetVT.SimpleTy != MVT::v4i64)
17836 return Register();
17837 if ((Subtarget->hasAVX2())) {
17838 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17839 }
17840 return Register();
17841}
17842
17843Register fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17844 switch (VT.SimpleTy) {
17845 case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op1);
17846 case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op1);
17847 case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op1);
17848 case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op1);
17849 case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op1);
17850 case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op1);
17851 case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op1);
17852 case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op1);
17853 default: return Register();
17854 }
17855}
17856
17857// FastEmit functions for X86ISD::PMULDQ.
17858
17859Register fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17860 if (RetVT.SimpleTy != MVT::v2i64)
17861 return Register();
17862 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17863 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17864 }
17865 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17866 return fastEmitInst_rr(MachineInstOpcode: X86::PMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17867 }
17868 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17869 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17870 }
17871 return Register();
17872}
17873
17874Register fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17875 if (RetVT.SimpleTy != MVT::v4i64)
17876 return Register();
17877 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17878 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17879 }
17880 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17881 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17882 }
17883 return Register();
17884}
17885
17886Register fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
17887 if (RetVT.SimpleTy != MVT::v8i64)
17888 return Register();
17889 if ((Subtarget->hasAVX512())) {
17890 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17891 }
17892 return Register();
17893}
17894
17895Register fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17896 switch (VT.SimpleTy) {
17897 case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17898 case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17899 case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
17900 default: return Register();
17901 }
17902}
17903
17904// FastEmit functions for X86ISD::PMULUDQ.
17905
17906Register fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17907 if (RetVT.SimpleTy != MVT::v2i64)
17908 return Register();
17909 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17910 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17911 }
17912 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17913 return fastEmitInst_rr(MachineInstOpcode: X86::PMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17914 }
17915 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17916 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17917 }
17918 return Register();
17919}
17920
17921Register fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17922 if (RetVT.SimpleTy != MVT::v4i64)
17923 return Register();
17924 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17925 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17926 }
17927 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17928 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17929 }
17930 return Register();
17931}
17932
17933Register fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
17934 if (RetVT.SimpleTy != MVT::v8i64)
17935 return Register();
17936 if ((Subtarget->hasAVX512())) {
17937 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17938 }
17939 return Register();
17940}
17941
17942Register fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17943 switch (VT.SimpleTy) {
17944 case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17945 case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17946 case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
17947 default: return Register();
17948 }
17949}
17950
17951// FastEmit functions for X86ISD::PSADBW.
17952
17953Register fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17954 if (RetVT.SimpleTy != MVT::v2i64)
17955 return Register();
17956 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17957 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17958 }
17959 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17960 return fastEmitInst_rr(MachineInstOpcode: X86::PSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17961 }
17962 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17963 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17964 }
17965 return Register();
17966}
17967
17968Register fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17969 if (RetVT.SimpleTy != MVT::v4i64)
17970 return Register();
17971 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17972 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17973 }
17974 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17975 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17976 }
17977 return Register();
17978}
17979
17980Register fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17981 if (RetVT.SimpleTy != MVT::v8i64)
17982 return Register();
17983 if ((Subtarget->hasBWI())) {
17984 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17985 }
17986 return Register();
17987}
17988
17989Register fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17990 switch (VT.SimpleTy) {
17991 case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op1);
17992 case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op1);
17993 case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op1);
17994 default: return Register();
17995 }
17996}
17997
17998// FastEmit functions for X86ISD::PSHUFB.
17999
18000Register fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18001 if (RetVT.SimpleTy != MVT::v16i8)
18002 return Register();
18003 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18004 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18005 }
18006 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
18007 return fastEmitInst_rr(MachineInstOpcode: X86::PSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
18008 }
18009 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18010 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
18011 }
18012 return Register();
18013}
18014
18015Register fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18016 if (RetVT.SimpleTy != MVT::v32i8)
18017 return Register();
18018 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18019 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18020 }
18021 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18022 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBYrr, RC: &X86::VR256RegClass, Op0, Op1);
18023 }
18024 return Register();
18025}
18026
18027Register fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
18028 if (RetVT.SimpleTy != MVT::v64i8)
18029 return Register();
18030 if ((Subtarget->hasBWI())) {
18031 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZrr, RC: &X86::VR512RegClass, Op0, Op1);
18032 }
18033 return Register();
18034}
18035
18036Register fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18037 switch (VT.SimpleTy) {
18038 case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op1);
18039 case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op1);
18040 case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op1);
18041 default: return Register();
18042 }
18043}
18044
18045// FastEmit functions for X86ISD::RCP14S.
18046
18047Register fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18048 if (RetVT.SimpleTy != MVT::v8f16)
18049 return Register();
18050 if ((Subtarget->hasFP16())) {
18051 return fastEmitInst_rr(MachineInstOpcode: X86::VRCPSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18052 }
18053 return Register();
18054}
18055
18056Register fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18057 if (RetVT.SimpleTy != MVT::v4f32)
18058 return Register();
18059 if ((Subtarget->hasAVX512())) {
18060 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18061 }
18062 return Register();
18063}
18064
18065Register fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18066 if (RetVT.SimpleTy != MVT::v2f64)
18067 return Register();
18068 if ((Subtarget->hasAVX512())) {
18069 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18070 }
18071 return Register();
18072}
18073
18074Register fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18075 switch (VT.SimpleTy) {
18076 case MVT::v8f16: return fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(RetVT, Op0, Op1);
18077 case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op1);
18078 case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op1);
18079 default: return Register();
18080 }
18081}
18082
18083// FastEmit functions for X86ISD::RSQRT14S.
18084
18085Register fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18086 if (RetVT.SimpleTy != MVT::v8f16)
18087 return Register();
18088 if ((Subtarget->hasFP16())) {
18089 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRTSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18090 }
18091 return Register();
18092}
18093
18094Register fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18095 if (RetVT.SimpleTy != MVT::v4f32)
18096 return Register();
18097 if ((Subtarget->hasAVX512())) {
18098 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18099 }
18100 return Register();
18101}
18102
18103Register fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18104 if (RetVT.SimpleTy != MVT::v2f64)
18105 return Register();
18106 if ((Subtarget->hasAVX512())) {
18107 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18108 }
18109 return Register();
18110}
18111
18112Register fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18113 switch (VT.SimpleTy) {
18114 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(RetVT, Op0, Op1);
18115 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op1);
18116 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op1);
18117 default: return Register();
18118 }
18119}
18120
18121// FastEmit functions for X86ISD::SCALEF.
18122
18123Register fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18124 if (RetVT.SimpleTy != MVT::v8f16)
18125 return Register();
18126 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18127 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18128 }
18129 return Register();
18130}
18131
18132Register fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18133 if (RetVT.SimpleTy != MVT::v16f16)
18134 return Register();
18135 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18136 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18137 }
18138 return Register();
18139}
18140
18141Register fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18142 if (RetVT.SimpleTy != MVT::v32f16)
18143 return Register();
18144 if ((Subtarget->hasFP16())) {
18145 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
18146 }
18147 return Register();
18148}
18149
18150Register fastEmit_X86ISD_SCALEF_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
18151 if (RetVT.SimpleTy != MVT::v8bf16)
18152 return Register();
18153 if ((Subtarget->hasAVX10_2())) {
18154 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18155 }
18156 return Register();
18157}
18158
18159Register fastEmit_X86ISD_SCALEF_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
18160 if (RetVT.SimpleTy != MVT::v16bf16)
18161 return Register();
18162 if ((Subtarget->hasAVX10_2())) {
18163 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18164 }
18165 return Register();
18166}
18167
18168Register fastEmit_X86ISD_SCALEF_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
18169 if (RetVT.SimpleTy != MVT::v32bf16)
18170 return Register();
18171 if ((Subtarget->hasAVX10_2())) {
18172 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
18173 }
18174 return Register();
18175}
18176
18177Register fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18178 if (RetVT.SimpleTy != MVT::v4f32)
18179 return Register();
18180 if ((Subtarget->hasVLX())) {
18181 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18182 }
18183 return Register();
18184}
18185
18186Register fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18187 if (RetVT.SimpleTy != MVT::v8f32)
18188 return Register();
18189 if ((Subtarget->hasVLX())) {
18190 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18191 }
18192 return Register();
18193}
18194
18195Register fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18196 if (RetVT.SimpleTy != MVT::v16f32)
18197 return Register();
18198 if ((Subtarget->hasAVX512())) {
18199 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18200 }
18201 return Register();
18202}
18203
18204Register fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18205 if (RetVT.SimpleTy != MVT::v2f64)
18206 return Register();
18207 if ((Subtarget->hasVLX())) {
18208 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18209 }
18210 return Register();
18211}
18212
18213Register fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18214 if (RetVT.SimpleTy != MVT::v4f64)
18215 return Register();
18216 if ((Subtarget->hasVLX())) {
18217 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18218 }
18219 return Register();
18220}
18221
18222Register fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18223 if (RetVT.SimpleTy != MVT::v8f64)
18224 return Register();
18225 if ((Subtarget->hasAVX512())) {
18226 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18227 }
18228 return Register();
18229}
18230
18231Register fastEmit_X86ISD_SCALEF_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18232 switch (VT.SimpleTy) {
18233 case MVT::v8f16: return fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(RetVT, Op0, Op1);
18234 case MVT::v16f16: return fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(RetVT, Op0, Op1);
18235 case MVT::v32f16: return fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(RetVT, Op0, Op1);
18236 case MVT::v8bf16: return fastEmit_X86ISD_SCALEF_MVT_v8bf16_rr(RetVT, Op0, Op1);
18237 case MVT::v16bf16: return fastEmit_X86ISD_SCALEF_MVT_v16bf16_rr(RetVT, Op0, Op1);
18238 case MVT::v32bf16: return fastEmit_X86ISD_SCALEF_MVT_v32bf16_rr(RetVT, Op0, Op1);
18239 case MVT::v4f32: return fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(RetVT, Op0, Op1);
18240 case MVT::v8f32: return fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(RetVT, Op0, Op1);
18241 case MVT::v16f32: return fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(RetVT, Op0, Op1);
18242 case MVT::v2f64: return fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(RetVT, Op0, Op1);
18243 case MVT::v4f64: return fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(RetVT, Op0, Op1);
18244 case MVT::v8f64: return fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(RetVT, Op0, Op1);
18245 default: return Register();
18246 }
18247}
18248
18249// FastEmit functions for X86ISD::SCALEFS.
18250
18251Register fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18252 if (RetVT.SimpleTy != MVT::v8f16)
18253 return Register();
18254 if ((Subtarget->hasFP16())) {
18255 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18256 }
18257 return Register();
18258}
18259
18260Register fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18261 if (RetVT.SimpleTy != MVT::v4f32)
18262 return Register();
18263 if ((Subtarget->hasAVX512())) {
18264 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18265 }
18266 return Register();
18267}
18268
18269Register fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18270 if (RetVT.SimpleTy != MVT::v2f64)
18271 return Register();
18272 if ((Subtarget->hasAVX512())) {
18273 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18274 }
18275 return Register();
18276}
18277
18278Register fastEmit_X86ISD_SCALEFS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18279 switch (VT.SimpleTy) {
18280 case MVT::v8f16: return fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(RetVT, Op0, Op1);
18281 case MVT::v4f32: return fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(RetVT, Op0, Op1);
18282 case MVT::v2f64: return fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(RetVT, Op0, Op1);
18283 default: return Register();
18284 }
18285}
18286
18287// FastEmit functions for X86ISD::STRICT_FP80_ADD.
18288
18289Register fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18290 if (RetVT.SimpleTy != MVT::f80)
18291 return Register();
18292 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
18293}
18294
18295Register fastEmit_X86ISD_STRICT_FP80_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18296 switch (VT.SimpleTy) {
18297 case MVT::f80: return fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
18298 default: return Register();
18299 }
18300}
18301
18302// FastEmit functions for X86ISD::UNPCKH.
18303
18304Register fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18305 if (RetVT.SimpleTy != MVT::v16i8)
18306 return Register();
18307 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18308 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18309 }
18310 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18311 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18312 }
18313 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18314 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18315 }
18316 return Register();
18317}
18318
18319Register fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18320 if (RetVT.SimpleTy != MVT::v32i8)
18321 return Register();
18322 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18323 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18324 }
18325 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18326 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
18327 }
18328 return Register();
18329}
18330
18331Register fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
18332 if (RetVT.SimpleTy != MVT::v64i8)
18333 return Register();
18334 if ((Subtarget->hasBWI())) {
18335 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
18336 }
18337 return Register();
18338}
18339
18340Register fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
18341 if (RetVT.SimpleTy != MVT::v8i16)
18342 return Register();
18343 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18344 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18345 }
18346 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18347 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18348 }
18349 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18350 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18351 }
18352 return Register();
18353}
18354
18355Register fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
18356 if (RetVT.SimpleTy != MVT::v16i16)
18357 return Register();
18358 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18359 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18360 }
18361 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18362 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18363 }
18364 return Register();
18365}
18366
18367Register fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
18368 if (RetVT.SimpleTy != MVT::v32i16)
18369 return Register();
18370 if ((Subtarget->hasBWI())) {
18371 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18372 }
18373 return Register();
18374}
18375
18376Register fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
18377 if (RetVT.SimpleTy != MVT::v4i32)
18378 return Register();
18379 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18380 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18381 }
18382 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18383 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18384 }
18385 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18386 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18387 }
18388 return Register();
18389}
18390
18391Register fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
18392 if (RetVT.SimpleTy != MVT::v8i32)
18393 return Register();
18394 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18395 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18396 }
18397 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18398 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18399 }
18400 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18401 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18402 }
18403 return Register();
18404}
18405
18406Register fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
18407 if (RetVT.SimpleTy != MVT::v16i32)
18408 return Register();
18409 if ((Subtarget->hasAVX512())) {
18410 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18411 }
18412 return Register();
18413}
18414
18415Register fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
18416 if (RetVT.SimpleTy != MVT::v2i64)
18417 return Register();
18418 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18419 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18420 }
18421 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18422 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18423 }
18424 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18425 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18426 }
18427 return Register();
18428}
18429
18430Register fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
18431 if (RetVT.SimpleTy != MVT::v4i64)
18432 return Register();
18433 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18434 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18435 }
18436 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18437 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18438 }
18439 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18440 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18441 }
18442 return Register();
18443}
18444
18445Register fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
18446 if (RetVT.SimpleTy != MVT::v8i64)
18447 return Register();
18448 if ((Subtarget->hasAVX512())) {
18449 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18450 }
18451 return Register();
18452}
18453
18454Register fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18455 if (RetVT.SimpleTy != MVT::v4f32)
18456 return Register();
18457 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18458 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18459 }
18460 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18461 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18462 }
18463 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18464 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18465 }
18466 return Register();
18467}
18468
18469Register fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18470 if (RetVT.SimpleTy != MVT::v8f32)
18471 return Register();
18472 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18473 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18474 }
18475 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18476 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18477 }
18478 return Register();
18479}
18480
18481Register fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18482 if (RetVT.SimpleTy != MVT::v16f32)
18483 return Register();
18484 if ((Subtarget->hasAVX512())) {
18485 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18486 }
18487 return Register();
18488}
18489
18490Register fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18491 if (RetVT.SimpleTy != MVT::v2f64)
18492 return Register();
18493 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18494 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18495 }
18496 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18497 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18498 }
18499 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18500 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18501 }
18502 return Register();
18503}
18504
18505Register fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18506 if (RetVT.SimpleTy != MVT::v4f64)
18507 return Register();
18508 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18509 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18510 }
18511 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18512 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18513 }
18514 return Register();
18515}
18516
18517Register fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18518 if (RetVT.SimpleTy != MVT::v8f64)
18519 return Register();
18520 if ((Subtarget->hasAVX512())) {
18521 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18522 }
18523 return Register();
18524}
18525
18526Register fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18527 switch (VT.SimpleTy) {
18528 case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op1);
18529 case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op1);
18530 case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op1);
18531 case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op1);
18532 case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op1);
18533 case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op1);
18534 case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op1);
18535 case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op1);
18536 case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op1);
18537 case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op1);
18538 case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op1);
18539 case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op1);
18540 case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op1);
18541 case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op1);
18542 case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op1);
18543 case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op1);
18544 case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op1);
18545 case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op1);
18546 default: return Register();
18547 }
18548}
18549
18550// FastEmit functions for X86ISD::UNPCKL.
18551
18552Register fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18553 if (RetVT.SimpleTy != MVT::v16i8)
18554 return Register();
18555 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18556 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18557 }
18558 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18559 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18560 }
18561 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18562 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18563 }
18564 return Register();
18565}
18566
18567Register fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18568 if (RetVT.SimpleTy != MVT::v32i8)
18569 return Register();
18570 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18571 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18572 }
18573 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18574 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
18575 }
18576 return Register();
18577}
18578
18579Register fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
18580 if (RetVT.SimpleTy != MVT::v64i8)
18581 return Register();
18582 if ((Subtarget->hasBWI())) {
18583 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
18584 }
18585 return Register();
18586}
18587
18588Register fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
18589 if (RetVT.SimpleTy != MVT::v8i16)
18590 return Register();
18591 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18592 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18593 }
18594 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18595 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18596 }
18597 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18598 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18599 }
18600 return Register();
18601}
18602
18603Register fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
18604 if (RetVT.SimpleTy != MVT::v16i16)
18605 return Register();
18606 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18607 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18608 }
18609 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18610 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18611 }
18612 return Register();
18613}
18614
18615Register fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
18616 if (RetVT.SimpleTy != MVT::v32i16)
18617 return Register();
18618 if ((Subtarget->hasBWI())) {
18619 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18620 }
18621 return Register();
18622}
18623
18624Register fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
18625 if (RetVT.SimpleTy != MVT::v4i32)
18626 return Register();
18627 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18628 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18629 }
18630 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18631 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18632 }
18633 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18634 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18635 }
18636 return Register();
18637}
18638
18639Register fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
18640 if (RetVT.SimpleTy != MVT::v8i32)
18641 return Register();
18642 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18643 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18644 }
18645 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18646 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18647 }
18648 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18649 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18650 }
18651 return Register();
18652}
18653
18654Register fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
18655 if (RetVT.SimpleTy != MVT::v16i32)
18656 return Register();
18657 if ((Subtarget->hasAVX512())) {
18658 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18659 }
18660 return Register();
18661}
18662
18663Register fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
18664 if (RetVT.SimpleTy != MVT::v2i64)
18665 return Register();
18666 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18667 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18668 }
18669 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18670 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18671 }
18672 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18673 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18674 }
18675 return Register();
18676}
18677
18678Register fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
18679 if (RetVT.SimpleTy != MVT::v4i64)
18680 return Register();
18681 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18682 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18683 }
18684 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18685 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18686 }
18687 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18688 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18689 }
18690 return Register();
18691}
18692
18693Register fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
18694 if (RetVT.SimpleTy != MVT::v8i64)
18695 return Register();
18696 if ((Subtarget->hasAVX512())) {
18697 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18698 }
18699 return Register();
18700}
18701
18702Register fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18703 if (RetVT.SimpleTy != MVT::v4f32)
18704 return Register();
18705 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18706 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18707 }
18708 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18709 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18710 }
18711 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18712 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18713 }
18714 return Register();
18715}
18716
18717Register fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18718 if (RetVT.SimpleTy != MVT::v8f32)
18719 return Register();
18720 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18721 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18722 }
18723 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18724 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18725 }
18726 return Register();
18727}
18728
18729Register fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18730 if (RetVT.SimpleTy != MVT::v16f32)
18731 return Register();
18732 if ((Subtarget->hasAVX512())) {
18733 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18734 }
18735 return Register();
18736}
18737
18738Register fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18739 if (RetVT.SimpleTy != MVT::v2f64)
18740 return Register();
18741 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18742 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18743 }
18744 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18745 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18746 }
18747 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18748 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18749 }
18750 return Register();
18751}
18752
18753Register fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18754 if (RetVT.SimpleTy != MVT::v4f64)
18755 return Register();
18756 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18757 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18758 }
18759 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18760 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18761 }
18762 return Register();
18763}
18764
18765Register fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18766 if (RetVT.SimpleTy != MVT::v8f64)
18767 return Register();
18768 if ((Subtarget->hasAVX512())) {
18769 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18770 }
18771 return Register();
18772}
18773
18774Register fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18775 switch (VT.SimpleTy) {
18776 case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op1);
18777 case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op1);
18778 case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op1);
18779 case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op1);
18780 case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op1);
18781 case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op1);
18782 case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op1);
18783 case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op1);
18784 case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op1);
18785 case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op1);
18786 case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op1);
18787 case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op1);
18788 case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op1);
18789 case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op1);
18790 case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op1);
18791 case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op1);
18792 case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op1);
18793 case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op1);
18794 default: return Register();
18795 }
18796}
18797
18798// FastEmit functions for X86ISD::VCVT2PH2BF8.
18799
18800Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18801 if (RetVT.SimpleTy != MVT::v16i8)
18802 return Register();
18803 if ((Subtarget->hasAVX10_2())) {
18804 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18805 }
18806 return Register();
18807}
18808
18809Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18810 if (RetVT.SimpleTy != MVT::v32i8)
18811 return Register();
18812 if ((Subtarget->hasAVX10_2())) {
18813 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18814 }
18815 return Register();
18816}
18817
18818Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18819 if (RetVT.SimpleTy != MVT::v64i8)
18820 return Register();
18821 if ((Subtarget->hasAVX10_2())) {
18822 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Zrr, RC: &X86::VR512RegClass, Op0, Op1);
18823 }
18824 return Register();
18825}
18826
18827Register fastEmit_X86ISD_VCVT2PH2BF8_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18828 switch (VT.SimpleTy) {
18829 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v8f16_rr(RetVT, Op0, Op1);
18830 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v16f16_rr(RetVT, Op0, Op1);
18831 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v32f16_rr(RetVT, Op0, Op1);
18832 default: return Register();
18833 }
18834}
18835
18836// FastEmit functions for X86ISD::VCVT2PH2BF8S.
18837
18838Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18839 if (RetVT.SimpleTy != MVT::v16i8)
18840 return Register();
18841 if ((Subtarget->hasAVX10_2())) {
18842 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18843 }
18844 return Register();
18845}
18846
18847Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18848 if (RetVT.SimpleTy != MVT::v32i8)
18849 return Register();
18850 if ((Subtarget->hasAVX10_2())) {
18851 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18852 }
18853 return Register();
18854}
18855
18856Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18857 if (RetVT.SimpleTy != MVT::v64i8)
18858 return Register();
18859 if ((Subtarget->hasAVX10_2())) {
18860 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZrr, RC: &X86::VR512RegClass, Op0, Op1);
18861 }
18862 return Register();
18863}
18864
18865Register fastEmit_X86ISD_VCVT2PH2BF8S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18866 switch (VT.SimpleTy) {
18867 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v8f16_rr(RetVT, Op0, Op1);
18868 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v16f16_rr(RetVT, Op0, Op1);
18869 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v32f16_rr(RetVT, Op0, Op1);
18870 default: return Register();
18871 }
18872}
18873
18874// FastEmit functions for X86ISD::VCVT2PH2HF8.
18875
18876Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18877 if (RetVT.SimpleTy != MVT::v16i8)
18878 return Register();
18879 if ((Subtarget->hasAVX10_2())) {
18880 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18881 }
18882 return Register();
18883}
18884
18885Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18886 if (RetVT.SimpleTy != MVT::v32i8)
18887 return Register();
18888 if ((Subtarget->hasAVX10_2())) {
18889 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18890 }
18891 return Register();
18892}
18893
18894Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18895 if (RetVT.SimpleTy != MVT::v64i8)
18896 return Register();
18897 if ((Subtarget->hasAVX10_2())) {
18898 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Zrr, RC: &X86::VR512RegClass, Op0, Op1);
18899 }
18900 return Register();
18901}
18902
18903Register fastEmit_X86ISD_VCVT2PH2HF8_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18904 switch (VT.SimpleTy) {
18905 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v8f16_rr(RetVT, Op0, Op1);
18906 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v16f16_rr(RetVT, Op0, Op1);
18907 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v32f16_rr(RetVT, Op0, Op1);
18908 default: return Register();
18909 }
18910}
18911
18912// FastEmit functions for X86ISD::VCVT2PH2HF8S.
18913
18914Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18915 if (RetVT.SimpleTy != MVT::v16i8)
18916 return Register();
18917 if ((Subtarget->hasAVX10_2())) {
18918 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18919 }
18920 return Register();
18921}
18922
18923Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18924 if (RetVT.SimpleTy != MVT::v32i8)
18925 return Register();
18926 if ((Subtarget->hasAVX10_2())) {
18927 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18928 }
18929 return Register();
18930}
18931
18932Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18933 if (RetVT.SimpleTy != MVT::v64i8)
18934 return Register();
18935 if ((Subtarget->hasAVX10_2())) {
18936 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZrr, RC: &X86::VR512RegClass, Op0, Op1);
18937 }
18938 return Register();
18939}
18940
18941Register fastEmit_X86ISD_VCVT2PH2HF8S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18942 switch (VT.SimpleTy) {
18943 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v8f16_rr(RetVT, Op0, Op1);
18944 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v16f16_rr(RetVT, Op0, Op1);
18945 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v32f16_rr(RetVT, Op0, Op1);
18946 default: return Register();
18947 }
18948}
18949
18950// FastEmit functions for X86ISD::VFCMULC.
18951
18952Register fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18953 if (RetVT.SimpleTy != MVT::v4f32)
18954 return Register();
18955 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18956 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18957 }
18958 return Register();
18959}
18960
18961Register fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18962 if (RetVT.SimpleTy != MVT::v8f32)
18963 return Register();
18964 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18965 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18966 }
18967 return Register();
18968}
18969
18970Register fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18971 if (RetVT.SimpleTy != MVT::v16f32)
18972 return Register();
18973 if ((Subtarget->hasFP16())) {
18974 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
18975 }
18976 return Register();
18977}
18978
18979Register fastEmit_X86ISD_VFCMULC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18980 switch (VT.SimpleTy) {
18981 case MVT::v4f32: return fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
18982 case MVT::v8f32: return fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
18983 case MVT::v16f32: return fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
18984 default: return Register();
18985 }
18986}
18987
18988// FastEmit functions for X86ISD::VFCMULCSH.
18989
18990Register fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18991 if (RetVT.SimpleTy != MVT::v4f32)
18992 return Register();
18993 if ((Subtarget->hasFP16())) {
18994 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18995 }
18996 return Register();
18997}
18998
18999Register fastEmit_X86ISD_VFCMULCSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19000 switch (VT.SimpleTy) {
19001 case MVT::v4f32: return fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
19002 default: return Register();
19003 }
19004}
19005
19006// FastEmit functions for X86ISD::VFMULC.
19007
19008Register fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19009 if (RetVT.SimpleTy != MVT::v4f32)
19010 return Register();
19011 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19012 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19013 }
19014 return Register();
19015}
19016
19017Register fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19018 if (RetVT.SimpleTy != MVT::v8f32)
19019 return Register();
19020 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19021 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19022 }
19023 return Register();
19024}
19025
19026Register fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19027 if (RetVT.SimpleTy != MVT::v16f32)
19028 return Register();
19029 if ((Subtarget->hasFP16())) {
19030 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
19031 }
19032 return Register();
19033}
19034
19035Register fastEmit_X86ISD_VFMULC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19036 switch (VT.SimpleTy) {
19037 case MVT::v4f32: return fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
19038 case MVT::v8f32: return fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
19039 case MVT::v16f32: return fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
19040 default: return Register();
19041 }
19042}
19043
19044// FastEmit functions for X86ISD::VFMULCSH.
19045
19046Register fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19047 if (RetVT.SimpleTy != MVT::v4f32)
19048 return Register();
19049 if ((Subtarget->hasFP16())) {
19050 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
19051 }
19052 return Register();
19053}
19054
19055Register fastEmit_X86ISD_VFMULCSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19056 switch (VT.SimpleTy) {
19057 case MVT::v4f32: return fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
19058 default: return Register();
19059 }
19060}
19061
19062// FastEmit functions for X86ISD::VFPROUND2.
19063
19064Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8f16_rr(Register Op0, Register Op1) {
19065 if ((Subtarget->hasAVX10_2())) {
19066 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19067 }
19068 return Register();
19069}
19070
19071Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8bf16_rr(Register Op0, Register Op1) {
19072 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
19073 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19074 }
19075 return Register();
19076}
19077
19078Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19079switch (RetVT.SimpleTy) {
19080 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8f16_rr(Op0, Op1);
19081 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8bf16_rr(Op0, Op1);
19082 default: return Register();
19083}
19084}
19085
19086Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16f16_rr(Register Op0, Register Op1) {
19087 if ((Subtarget->hasAVX10_2())) {
19088 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19089 }
19090 return Register();
19091}
19092
19093Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16bf16_rr(Register Op0, Register Op1) {
19094 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
19095 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19096 }
19097 return Register();
19098}
19099
19100Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19101switch (RetVT.SimpleTy) {
19102 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16f16_rr(Op0, Op1);
19103 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16bf16_rr(Op0, Op1);
19104 default: return Register();
19105}
19106}
19107
19108Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32f16_rr(Register Op0, Register Op1) {
19109 if ((Subtarget->hasAVX10_2())) {
19110 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZrr, RC: &X86::VR512RegClass, Op0, Op1);
19111 }
19112 return Register();
19113}
19114
19115Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32bf16_rr(Register Op0, Register Op1) {
19116 if ((Subtarget->hasBF16())) {
19117 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19118 }
19119 return Register();
19120}
19121
19122Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19123switch (RetVT.SimpleTy) {
19124 case MVT::v32f16: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32f16_rr(Op0, Op1);
19125 case MVT::v32bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32bf16_rr(Op0, Op1);
19126 default: return Register();
19127}
19128}
19129
19130Register fastEmit_X86ISD_VFPROUND2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19131 switch (VT.SimpleTy) {
19132 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_rr(RetVT, Op0, Op1);
19133 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_rr(RetVT, Op0, Op1);
19134 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_rr(RetVT, Op0, Op1);
19135 default: return Register();
19136 }
19137}
19138
19139// FastEmit functions for X86ISD::VP2INTERSECT.
19140
19141Register fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19142 if (RetVT.SimpleTy != MVT::Untyped)
19143 return Register();
19144 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19145 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ128rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
19146 }
19147 return Register();
19148}
19149
19150Register fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19151 if (RetVT.SimpleTy != MVT::Untyped)
19152 return Register();
19153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19154 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ256rr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
19155 }
19156 return Register();
19157}
19158
19159Register fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19160 if (RetVT.SimpleTy != MVT::Untyped)
19161 return Register();
19162 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
19163 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZrr, RC: &X86::VK16PAIRRegClass, Op0, Op1);
19164 }
19165 return Register();
19166}
19167
19168Register fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19169 if (RetVT.SimpleTy != MVT::Untyped)
19170 return Register();
19171 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19172 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ128rr, RC: &X86::VK2PAIRRegClass, Op0, Op1);
19173 }
19174 return Register();
19175}
19176
19177Register fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19178 if (RetVT.SimpleTy != MVT::Untyped)
19179 return Register();
19180 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19181 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ256rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
19182 }
19183 return Register();
19184}
19185
19186Register fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19187 if (RetVT.SimpleTy != MVT::Untyped)
19188 return Register();
19189 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
19190 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZrr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
19191 }
19192 return Register();
19193}
19194
19195Register fastEmit_X86ISD_VP2INTERSECT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19196 switch (VT.SimpleTy) {
19197 case MVT::v4i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(RetVT, Op0, Op1);
19198 case MVT::v8i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(RetVT, Op0, Op1);
19199 case MVT::v16i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(RetVT, Op0, Op1);
19200 case MVT::v2i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(RetVT, Op0, Op1);
19201 case MVT::v4i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(RetVT, Op0, Op1);
19202 case MVT::v8i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(RetVT, Op0, Op1);
19203 default: return Register();
19204 }
19205}
19206
19207// FastEmit functions for X86ISD::VPERMV.
19208
19209Register fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19210 if (RetVT.SimpleTy != MVT::v16i8)
19211 return Register();
19212 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
19213 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19214 }
19215 return Register();
19216}
19217
19218Register fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19219 if (RetVT.SimpleTy != MVT::v32i8)
19220 return Register();
19221 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
19222 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19223 }
19224 return Register();
19225}
19226
19227Register fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19228 if (RetVT.SimpleTy != MVT::v64i8)
19229 return Register();
19230 if ((Subtarget->hasVBMI())) {
19231 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZrr, RC: &X86::VR512RegClass, Op0, Op1);
19232 }
19233 return Register();
19234}
19235
19236Register fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19237 if (RetVT.SimpleTy != MVT::v8i16)
19238 return Register();
19239 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19240 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19241 }
19242 return Register();
19243}
19244
19245Register fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19246 if (RetVT.SimpleTy != MVT::v16i16)
19247 return Register();
19248 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19249 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19250 }
19251 return Register();
19252}
19253
19254Register fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19255 if (RetVT.SimpleTy != MVT::v32i16)
19256 return Register();
19257 if ((Subtarget->hasBWI())) {
19258 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19259 }
19260 return Register();
19261}
19262
19263Register fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19264 if (RetVT.SimpleTy != MVT::v8i32)
19265 return Register();
19266 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19267 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19268 }
19269 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19270 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19271 }
19272 return Register();
19273}
19274
19275Register fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19276 if (RetVT.SimpleTy != MVT::v16i32)
19277 return Register();
19278 if ((Subtarget->hasAVX512())) {
19279 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19280 }
19281 return Register();
19282}
19283
19284Register fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19285 if (RetVT.SimpleTy != MVT::v4i64)
19286 return Register();
19287 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19288 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19289 }
19290 return Register();
19291}
19292
19293Register fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19294 if (RetVT.SimpleTy != MVT::v8i64)
19295 return Register();
19296 if ((Subtarget->hasAVX512())) {
19297 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19298 }
19299 return Register();
19300}
19301
19302Register fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19303 switch (VT.SimpleTy) {
19304 case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op1);
19305 case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op1);
19306 case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op1);
19307 case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op1);
19308 case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op1);
19309 case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op1);
19310 case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op1);
19311 case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op1);
19312 case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op1);
19313 case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op1);
19314 default: return Register();
19315 }
19316}
19317
19318// FastEmit functions for X86ISD::VPMADDUBSW.
19319
19320Register fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19321 if (RetVT.SimpleTy != MVT::v8i16)
19322 return Register();
19323 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19324 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19325 }
19326 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
19327 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
19328 }
19329 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19330 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
19331 }
19332 return Register();
19333}
19334
19335Register fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19336 if (RetVT.SimpleTy != MVT::v16i16)
19337 return Register();
19338 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19339 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19340 }
19341 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19342 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
19343 }
19344 return Register();
19345}
19346
19347Register fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19348 if (RetVT.SimpleTy != MVT::v32i16)
19349 return Register();
19350 if ((Subtarget->hasBWI())) {
19351 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19352 }
19353 return Register();
19354}
19355
19356Register fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19357 switch (VT.SimpleTy) {
19358 case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op1);
19359 case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op1);
19360 case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op1);
19361 default: return Register();
19362 }
19363}
19364
19365// FastEmit functions for X86ISD::VPMADDWD.
19366
19367Register fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19368 if (RetVT.SimpleTy != MVT::v4i32)
19369 return Register();
19370 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19371 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19372 }
19373 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19374 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19375 }
19376 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19377 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19378 }
19379 return Register();
19380}
19381
19382Register fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19383 if (RetVT.SimpleTy != MVT::v8i32)
19384 return Register();
19385 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19386 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19387 }
19388 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19389 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19390 }
19391 return Register();
19392}
19393
19394Register fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19395 if (RetVT.SimpleTy != MVT::v16i32)
19396 return Register();
19397 if ((Subtarget->hasBWI())) {
19398 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19399 }
19400 return Register();
19401}
19402
19403Register fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19404 switch (VT.SimpleTy) {
19405 case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op1);
19406 case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op1);
19407 case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op1);
19408 default: return Register();
19409 }
19410}
19411
19412// FastEmit functions for X86ISD::VPSHA.
19413
19414Register fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19415 if (RetVT.SimpleTy != MVT::v16i8)
19416 return Register();
19417 if ((Subtarget->hasXOP())) {
19418 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHABrr, RC: &X86::VR128RegClass, Op0, Op1);
19419 }
19420 return Register();
19421}
19422
19423Register fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19424 if (RetVT.SimpleTy != MVT::v8i16)
19425 return Register();
19426 if ((Subtarget->hasXOP())) {
19427 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAWrr, RC: &X86::VR128RegClass, Op0, Op1);
19428 }
19429 return Register();
19430}
19431
19432Register fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19433 if (RetVT.SimpleTy != MVT::v4i32)
19434 return Register();
19435 if ((Subtarget->hasXOP())) {
19436 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHADrr, RC: &X86::VR128RegClass, Op0, Op1);
19437 }
19438 return Register();
19439}
19440
19441Register fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19442 if (RetVT.SimpleTy != MVT::v2i64)
19443 return Register();
19444 if ((Subtarget->hasXOP())) {
19445 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAQrr, RC: &X86::VR128RegClass, Op0, Op1);
19446 }
19447 return Register();
19448}
19449
19450Register fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19451 switch (VT.SimpleTy) {
19452 case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op1);
19453 case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op1);
19454 case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op1);
19455 case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op1);
19456 default: return Register();
19457 }
19458}
19459
19460// FastEmit functions for X86ISD::VPSHL.
19461
19462Register fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19463 if (RetVT.SimpleTy != MVT::v16i8)
19464 return Register();
19465 if ((Subtarget->hasXOP())) {
19466 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLBrr, RC: &X86::VR128RegClass, Op0, Op1);
19467 }
19468 return Register();
19469}
19470
19471Register fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19472 if (RetVT.SimpleTy != MVT::v8i16)
19473 return Register();
19474 if ((Subtarget->hasXOP())) {
19475 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19476 }
19477 return Register();
19478}
19479
19480Register fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19481 if (RetVT.SimpleTy != MVT::v4i32)
19482 return Register();
19483 if ((Subtarget->hasXOP())) {
19484 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19485 }
19486 return Register();
19487}
19488
19489Register fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19490 if (RetVT.SimpleTy != MVT::v2i64)
19491 return Register();
19492 if ((Subtarget->hasXOP())) {
19493 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLQrr, RC: &X86::VR128RegClass, Op0, Op1);
19494 }
19495 return Register();
19496}
19497
19498Register fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19499 switch (VT.SimpleTy) {
19500 case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op1);
19501 case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
19502 case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
19503 case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
19504 default: return Register();
19505 }
19506}
19507
19508// FastEmit functions for X86ISD::VPSHUFBITQMB.
19509
19510Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19511 if (RetVT.SimpleTy != MVT::v16i1)
19512 return Register();
19513 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
19514 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ128rr, RC: &X86::VK16RegClass, Op0, Op1);
19515 }
19516 return Register();
19517}
19518
19519Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19520 if (RetVT.SimpleTy != MVT::v32i1)
19521 return Register();
19522 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
19523 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ256rr, RC: &X86::VK32RegClass, Op0, Op1);
19524 }
19525 return Register();
19526}
19527
19528Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19529 if (RetVT.SimpleTy != MVT::v64i1)
19530 return Register();
19531 if ((Subtarget->hasBITALG())) {
19532 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZrr, RC: &X86::VK64RegClass, Op0, Op1);
19533 }
19534 return Register();
19535}
19536
19537Register fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19538 switch (VT.SimpleTy) {
19539 case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op1);
19540 case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op1);
19541 case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op1);
19542 default: return Register();
19543 }
19544}
19545
19546// FastEmit functions for X86ISD::VSHL.
19547
19548Register fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19549 if (RetVT.SimpleTy != MVT::v8i16)
19550 return Register();
19551 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19552 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19553 }
19554 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19555 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19556 }
19557 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19558 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19559 }
19560 return Register();
19561}
19562
19563Register fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19564 if (RetVT.SimpleTy != MVT::v4i32)
19565 return Register();
19566 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19567 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19568 }
19569 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19570 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19571 }
19572 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19573 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19574 }
19575 return Register();
19576}
19577
19578Register fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19579 if (RetVT.SimpleTy != MVT::v2i64)
19580 return Register();
19581 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19582 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19583 }
19584 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19585 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
19586 }
19587 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19588 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
19589 }
19590 return Register();
19591}
19592
19593Register fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19594 switch (VT.SimpleTy) {
19595 case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
19596 case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
19597 case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
19598 default: return Register();
19599 }
19600}
19601
19602// FastEmit functions for X86ISD::VSHLV.
19603
19604Register fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19605 if (RetVT.SimpleTy != MVT::v8i16)
19606 return Register();
19607 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19608 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19609 }
19610 return Register();
19611}
19612
19613Register fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19614 if (RetVT.SimpleTy != MVT::v16i16)
19615 return Register();
19616 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19617 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19618 }
19619 return Register();
19620}
19621
19622Register fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19623 if (RetVT.SimpleTy != MVT::v32i16)
19624 return Register();
19625 if ((Subtarget->hasBWI())) {
19626 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19627 }
19628 return Register();
19629}
19630
19631Register fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19632 if (RetVT.SimpleTy != MVT::v4i32)
19633 return Register();
19634 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19635 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19636 }
19637 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19638 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
19639 }
19640 return Register();
19641}
19642
19643Register fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19644 if (RetVT.SimpleTy != MVT::v8i32)
19645 return Register();
19646 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19647 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19648 }
19649 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19650 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19651 }
19652 return Register();
19653}
19654
19655Register fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19656 if (RetVT.SimpleTy != MVT::v16i32)
19657 return Register();
19658 if ((Subtarget->hasAVX512())) {
19659 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19660 }
19661 return Register();
19662}
19663
19664Register fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19665 if (RetVT.SimpleTy != MVT::v2i64)
19666 return Register();
19667 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19668 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19669 }
19670 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19671 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
19672 }
19673 return Register();
19674}
19675
19676Register fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19677 if (RetVT.SimpleTy != MVT::v4i64)
19678 return Register();
19679 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19680 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19681 }
19682 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19683 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
19684 }
19685 return Register();
19686}
19687
19688Register fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19689 if (RetVT.SimpleTy != MVT::v8i64)
19690 return Register();
19691 if ((Subtarget->hasAVX512())) {
19692 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19693 }
19694 return Register();
19695}
19696
19697Register fastEmit_X86ISD_VSHLV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19698 switch (VT.SimpleTy) {
19699 case MVT::v8i16: return fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(RetVT, Op0, Op1);
19700 case MVT::v16i16: return fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(RetVT, Op0, Op1);
19701 case MVT::v32i16: return fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(RetVT, Op0, Op1);
19702 case MVT::v4i32: return fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(RetVT, Op0, Op1);
19703 case MVT::v8i32: return fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(RetVT, Op0, Op1);
19704 case MVT::v16i32: return fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(RetVT, Op0, Op1);
19705 case MVT::v2i64: return fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(RetVT, Op0, Op1);
19706 case MVT::v4i64: return fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(RetVT, Op0, Op1);
19707 case MVT::v8i64: return fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(RetVT, Op0, Op1);
19708 default: return Register();
19709 }
19710}
19711
19712// FastEmit functions for X86ISD::VSRA.
19713
19714Register fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19715 if (RetVT.SimpleTy != MVT::v8i16)
19716 return Register();
19717 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19718 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19719 }
19720 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19721 return fastEmitInst_rr(MachineInstOpcode: X86::PSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
19722 }
19723 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19724 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
19725 }
19726 return Register();
19727}
19728
19729Register fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19730 if (RetVT.SimpleTy != MVT::v4i32)
19731 return Register();
19732 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19733 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19734 }
19735 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19736 return fastEmitInst_rr(MachineInstOpcode: X86::PSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
19737 }
19738 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19739 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
19740 }
19741 return Register();
19742}
19743
19744Register fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19745 if (RetVT.SimpleTy != MVT::v2i64)
19746 return Register();
19747 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19748 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19749 }
19750 return Register();
19751}
19752
19753Register fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19754 switch (VT.SimpleTy) {
19755 case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op1);
19756 case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op1);
19757 case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op1);
19758 default: return Register();
19759 }
19760}
19761
19762// FastEmit functions for X86ISD::VSRAV.
19763
19764Register fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19765 if (RetVT.SimpleTy != MVT::v8i16)
19766 return Register();
19767 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19768 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19769 }
19770 return Register();
19771}
19772
19773Register fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19774 if (RetVT.SimpleTy != MVT::v16i16)
19775 return Register();
19776 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19777 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19778 }
19779 return Register();
19780}
19781
19782Register fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19783 if (RetVT.SimpleTy != MVT::v32i16)
19784 return Register();
19785 if ((Subtarget->hasBWI())) {
19786 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19787 }
19788 return Register();
19789}
19790
19791Register fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19792 if (RetVT.SimpleTy != MVT::v4i32)
19793 return Register();
19794 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19795 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19796 }
19797 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19798 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDrr, RC: &X86::VR128RegClass, Op0, Op1);
19799 }
19800 return Register();
19801}
19802
19803Register fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19804 if (RetVT.SimpleTy != MVT::v8i32)
19805 return Register();
19806 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19807 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19808 }
19809 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19810 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19811 }
19812 return Register();
19813}
19814
19815Register fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19816 if (RetVT.SimpleTy != MVT::v16i32)
19817 return Register();
19818 if ((Subtarget->hasAVX512())) {
19819 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19820 }
19821 return Register();
19822}
19823
19824Register fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19825 if (RetVT.SimpleTy != MVT::v2i64)
19826 return Register();
19827 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19828 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19829 }
19830 return Register();
19831}
19832
19833Register fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19834 if (RetVT.SimpleTy != MVT::v4i64)
19835 return Register();
19836 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19837 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19838 }
19839 return Register();
19840}
19841
19842Register fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19843 if (RetVT.SimpleTy != MVT::v8i64)
19844 return Register();
19845 if ((Subtarget->hasAVX512())) {
19846 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19847 }
19848 return Register();
19849}
19850
19851Register fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19852 switch (VT.SimpleTy) {
19853 case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op1);
19854 case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op1);
19855 case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op1);
19856 case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op1);
19857 case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op1);
19858 case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op1);
19859 case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op1);
19860 case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op1);
19861 case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op1);
19862 default: return Register();
19863 }
19864}
19865
19866// FastEmit functions for X86ISD::VSRL.
19867
19868Register fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19869 if (RetVT.SimpleTy != MVT::v8i16)
19870 return Register();
19871 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19872 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19873 }
19874 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19875 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19876 }
19877 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19878 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19879 }
19880 return Register();
19881}
19882
19883Register fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19884 if (RetVT.SimpleTy != MVT::v4i32)
19885 return Register();
19886 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19887 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19888 }
19889 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19890 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19891 }
19892 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19893 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19894 }
19895 return Register();
19896}
19897
19898Register fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19899 if (RetVT.SimpleTy != MVT::v2i64)
19900 return Register();
19901 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19902 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19903 }
19904 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19905 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
19906 }
19907 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19908 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
19909 }
19910 return Register();
19911}
19912
19913Register fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19914 switch (VT.SimpleTy) {
19915 case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op1);
19916 case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op1);
19917 case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op1);
19918 default: return Register();
19919 }
19920}
19921
19922// FastEmit functions for X86ISD::VSRLV.
19923
19924Register fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19925 if (RetVT.SimpleTy != MVT::v8i16)
19926 return Register();
19927 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19928 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19929 }
19930 return Register();
19931}
19932
19933Register fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19934 if (RetVT.SimpleTy != MVT::v16i16)
19935 return Register();
19936 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19937 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19938 }
19939 return Register();
19940}
19941
19942Register fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19943 if (RetVT.SimpleTy != MVT::v32i16)
19944 return Register();
19945 if ((Subtarget->hasBWI())) {
19946 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19947 }
19948 return Register();
19949}
19950
19951Register fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19952 if (RetVT.SimpleTy != MVT::v4i32)
19953 return Register();
19954 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19955 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19956 }
19957 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19958 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
19959 }
19960 return Register();
19961}
19962
19963Register fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19964 if (RetVT.SimpleTy != MVT::v8i32)
19965 return Register();
19966 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19967 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19968 }
19969 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19970 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19971 }
19972 return Register();
19973}
19974
19975Register fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19976 if (RetVT.SimpleTy != MVT::v16i32)
19977 return Register();
19978 if ((Subtarget->hasAVX512())) {
19979 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19980 }
19981 return Register();
19982}
19983
19984Register fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19985 if (RetVT.SimpleTy != MVT::v2i64)
19986 return Register();
19987 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19988 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19989 }
19990 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19991 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
19992 }
19993 return Register();
19994}
19995
19996Register fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19997 if (RetVT.SimpleTy != MVT::v4i64)
19998 return Register();
19999 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20000 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20001 }
20002 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20003 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
20004 }
20005 return Register();
20006}
20007
20008Register fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20009 if (RetVT.SimpleTy != MVT::v8i64)
20010 return Register();
20011 if ((Subtarget->hasAVX512())) {
20012 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20013 }
20014 return Register();
20015}
20016
20017Register fastEmit_X86ISD_VSRLV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20018 switch (VT.SimpleTy) {
20019 case MVT::v8i16: return fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(RetVT, Op0, Op1);
20020 case MVT::v16i16: return fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(RetVT, Op0, Op1);
20021 case MVT::v32i16: return fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(RetVT, Op0, Op1);
20022 case MVT::v4i32: return fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(RetVT, Op0, Op1);
20023 case MVT::v8i32: return fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(RetVT, Op0, Op1);
20024 case MVT::v16i32: return fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(RetVT, Op0, Op1);
20025 case MVT::v2i64: return fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(RetVT, Op0, Op1);
20026 case MVT::v4i64: return fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(RetVT, Op0, Op1);
20027 case MVT::v8i64: return fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(RetVT, Op0, Op1);
20028 default: return Register();
20029 }
20030}
20031
20032// Top-level FastEmit function.
20033
20034Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
20035 switch (Opcode) {
20036 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
20037 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
20038 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
20039 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
20040 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
20041 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
20042 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
20043 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
20044 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
20045 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
20046 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
20047 case ISD::PDEP: return fastEmit_ISD_PDEP_rr(VT, RetVT, Op0, Op1);
20048 case ISD::PEXT: return fastEmit_ISD_PEXT_rr(VT, RetVT, Op0, Op1);
20049 case ISD::PSEUDO_FMAX: return fastEmit_ISD_PSEUDO_FMAX_rr(VT, RetVT, Op0, Op1);
20050 case ISD::PSEUDO_FMIN: return fastEmit_ISD_PSEUDO_FMIN_rr(VT, RetVT, Op0, Op1);
20051 case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op1);
20052 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
20053 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
20054 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
20055 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
20056 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
20057 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
20058 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
20059 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
20060 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
20061 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
20062 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
20063 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
20064 case ISD::STRICT_PSEUDO_FMAX: return fastEmit_ISD_STRICT_PSEUDO_FMAX_rr(VT, RetVT, Op0, Op1);
20065 case ISD::STRICT_PSEUDO_FMIN: return fastEmit_ISD_STRICT_PSEUDO_FMIN_rr(VT, RetVT, Op0, Op1);
20066 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
20067 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
20068 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
20069 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
20070 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
20071 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
20072 case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op1);
20073 case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op1);
20074 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op1);
20075 case X86ISD::BZHI: return fastEmit_X86ISD_BZHI_rr(VT, RetVT, Op0, Op1);
20076 case X86ISD::FADDS: return fastEmit_X86ISD_FADDS_rr(VT, RetVT, Op0, Op1);
20077 case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op1);
20078 case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op1);
20079 case X86ISD::FDIVS: return fastEmit_X86ISD_FDIVS_rr(VT, RetVT, Op0, Op1);
20080 case X86ISD::FGETEXPS: return fastEmit_X86ISD_FGETEXPS_rr(VT, RetVT, Op0, Op1);
20081 case X86ISD::FGETEXPS_SAE: return fastEmit_X86ISD_FGETEXPS_SAE_rr(VT, RetVT, Op0, Op1);
20082 case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op1);
20083 case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op1);
20084 case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op1);
20085 case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op1);
20086 case X86ISD::FMAXS_SAE: return fastEmit_X86ISD_FMAXS_SAE_rr(VT, RetVT, Op0, Op1);
20087 case X86ISD::FMAX_SAE: return fastEmit_X86ISD_FMAX_SAE_rr(VT, RetVT, Op0, Op1);
20088 case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op1);
20089 case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op1);
20090 case X86ISD::FMINS_SAE: return fastEmit_X86ISD_FMINS_SAE_rr(VT, RetVT, Op0, Op1);
20091 case X86ISD::FMIN_SAE: return fastEmit_X86ISD_FMIN_SAE_rr(VT, RetVT, Op0, Op1);
20092 case X86ISD::FMULS: return fastEmit_X86ISD_FMULS_rr(VT, RetVT, Op0, Op1);
20093 case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op1);
20094 case X86ISD::FP80_ADD: return fastEmit_X86ISD_FP80_ADD_rr(VT, RetVT, Op0, Op1);
20095 case X86ISD::FSQRTS: return fastEmit_X86ISD_FSQRTS_rr(VT, RetVT, Op0, Op1);
20096 case X86ISD::FSUBS: return fastEmit_X86ISD_FSUBS_rr(VT, RetVT, Op0, Op1);
20097 case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op1);
20098 case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op1);
20099 case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op1);
20100 case X86ISD::HADDS: return fastEmit_X86ISD_HADDS_rr(VT, RetVT, Op0, Op1);
20101 case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op1);
20102 case X86ISD::HSUBS: return fastEmit_X86ISD_HSUBS_rr(VT, RetVT, Op0, Op1);
20103 case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op1);
20104 case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op1);
20105 case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op1);
20106 case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op1);
20107 case X86ISD::MOVSH: return fastEmit_X86ISD_MOVSH_rr(VT, RetVT, Op0, Op1);
20108 case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op1);
20109 case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op1);
20110 case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op1);
20111 case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op1);
20112 case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op1);
20113 case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op1);
20114 case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op1);
20115 case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op1);
20116 case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op1);
20117 case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op1);
20118 case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op1);
20119 case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op1);
20120 case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op1);
20121 case X86ISD::SCALEF: return fastEmit_X86ISD_SCALEF_rr(VT, RetVT, Op0, Op1);
20122 case X86ISD::SCALEFS: return fastEmit_X86ISD_SCALEFS_rr(VT, RetVT, Op0, Op1);
20123 case X86ISD::STRICT_FP80_ADD: return fastEmit_X86ISD_STRICT_FP80_ADD_rr(VT, RetVT, Op0, Op1);
20124 case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op1);
20125 case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op1);
20126 case X86ISD::VCVT2PH2BF8: return fastEmit_X86ISD_VCVT2PH2BF8_rr(VT, RetVT, Op0, Op1);
20127 case X86ISD::VCVT2PH2BF8S: return fastEmit_X86ISD_VCVT2PH2BF8S_rr(VT, RetVT, Op0, Op1);
20128 case X86ISD::VCVT2PH2HF8: return fastEmit_X86ISD_VCVT2PH2HF8_rr(VT, RetVT, Op0, Op1);
20129 case X86ISD::VCVT2PH2HF8S: return fastEmit_X86ISD_VCVT2PH2HF8S_rr(VT, RetVT, Op0, Op1);
20130 case X86ISD::VFCMULC: return fastEmit_X86ISD_VFCMULC_rr(VT, RetVT, Op0, Op1);
20131 case X86ISD::VFCMULCSH: return fastEmit_X86ISD_VFCMULCSH_rr(VT, RetVT, Op0, Op1);
20132 case X86ISD::VFMULC: return fastEmit_X86ISD_VFMULC_rr(VT, RetVT, Op0, Op1);
20133 case X86ISD::VFMULCSH: return fastEmit_X86ISD_VFMULCSH_rr(VT, RetVT, Op0, Op1);
20134 case X86ISD::VFPROUND2: return fastEmit_X86ISD_VFPROUND2_rr(VT, RetVT, Op0, Op1);
20135 case X86ISD::VP2INTERSECT: return fastEmit_X86ISD_VP2INTERSECT_rr(VT, RetVT, Op0, Op1);
20136 case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op1);
20137 case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op1);
20138 case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op1);
20139 case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op1);
20140 case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op1);
20141 case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op1);
20142 case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op1);
20143 case X86ISD::VSHLV: return fastEmit_X86ISD_VSHLV_rr(VT, RetVT, Op0, Op1);
20144 case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op1);
20145 case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op1);
20146 case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op1);
20147 case X86ISD::VSRLV: return fastEmit_X86ISD_VSRLV_rr(VT, RetVT, Op0, Op1);
20148 default: return Register();
20149 }
20150}
20151
20152// FastEmit functions for ISD::ADD.
20153
20154Register fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20155 if (RetVT.SimpleTy != MVT::i8)
20156 return Register();
20157 if ((Subtarget->hasNDD())) {
20158 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20159 }
20160 if ((!Subtarget->hasNDD())) {
20161 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20162 }
20163 return Register();
20164}
20165
20166Register fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20167 if (RetVT.SimpleTy != MVT::i16)
20168 return Register();
20169 if ((Subtarget->hasNDD())) {
20170 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20171 }
20172 if ((!Subtarget->hasNDD())) {
20173 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20174 }
20175 return Register();
20176}
20177
20178Register fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20179 if (RetVT.SimpleTy != MVT::i32)
20180 return Register();
20181 if ((Subtarget->hasNDD())) {
20182 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20183 }
20184 if ((!Subtarget->hasNDD())) {
20185 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20186 }
20187 return Register();
20188}
20189
20190Register fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20191 switch (VT.SimpleTy) {
20192 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, imm1);
20193 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, imm1);
20194 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, imm1);
20195 default: return Register();
20196 }
20197}
20198
20199// FastEmit functions for ISD::AND.
20200
20201Register fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20202 if (RetVT.SimpleTy != MVT::i8)
20203 return Register();
20204 if ((Subtarget->hasNDD())) {
20205 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20206 }
20207 if ((!Subtarget->hasNDD())) {
20208 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20209 }
20210 return Register();
20211}
20212
20213Register fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20214 if (RetVT.SimpleTy != MVT::i16)
20215 return Register();
20216 if ((Subtarget->hasNDD())) {
20217 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20218 }
20219 if ((!Subtarget->hasNDD())) {
20220 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20221 }
20222 return Register();
20223}
20224
20225Register fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20226 if (RetVT.SimpleTy != MVT::i32)
20227 return Register();
20228 if ((Subtarget->hasNDD())) {
20229 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20230 }
20231 if ((!Subtarget->hasNDD())) {
20232 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20233 }
20234 return Register();
20235}
20236
20237Register fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20238 switch (VT.SimpleTy) {
20239 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, imm1);
20240 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, imm1);
20241 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, imm1);
20242 default: return Register();
20243 }
20244}
20245
20246// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
20247
20248Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20249 if (RetVT.SimpleTy != MVT::i32)
20250 return Register();
20251 if ((Subtarget->hasDQI())) {
20252 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDZrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20253 }
20254 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
20255 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRDrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20256 }
20257 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
20258 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20259 }
20260 return Register();
20261}
20262
20263Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20264 if (RetVT.SimpleTy != MVT::i64)
20265 return Register();
20266 if ((Subtarget->hasDQI())) {
20267 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQZrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20268 }
20269 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
20270 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRQrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20271 }
20272 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
20273 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20274 }
20275 return Register();
20276}
20277
20278Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20279 switch (VT.SimpleTy) {
20280 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, imm1);
20281 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, imm1);
20282 default: return Register();
20283 }
20284}
20285
20286// FastEmit functions for ISD::MUL.
20287
20288Register fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20289 if (RetVT.SimpleTy != MVT::i16)
20290 return Register();
20291 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL16rri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20292}
20293
20294Register fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20295 if (RetVT.SimpleTy != MVT::i32)
20296 return Register();
20297 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL32rri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20298}
20299
20300Register fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20301 switch (VT.SimpleTy) {
20302 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, imm1);
20303 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, imm1);
20304 default: return Register();
20305 }
20306}
20307
20308// FastEmit functions for ISD::OR.
20309
20310Register fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20311 if (RetVT.SimpleTy != MVT::i8)
20312 return Register();
20313 if ((Subtarget->hasNDD())) {
20314 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20315 }
20316 if ((!Subtarget->hasNDD())) {
20317 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20318 }
20319 return Register();
20320}
20321
20322Register fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20323 if (RetVT.SimpleTy != MVT::i16)
20324 return Register();
20325 if ((Subtarget->hasNDD())) {
20326 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20327 }
20328 if ((!Subtarget->hasNDD())) {
20329 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20330 }
20331 return Register();
20332}
20333
20334Register fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20335 if (RetVT.SimpleTy != MVT::i32)
20336 return Register();
20337 if ((Subtarget->hasNDD())) {
20338 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20339 }
20340 if ((!Subtarget->hasNDD())) {
20341 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20342 }
20343 return Register();
20344}
20345
20346Register fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20347 switch (VT.SimpleTy) {
20348 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, imm1);
20349 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, imm1);
20350 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, imm1);
20351 default: return Register();
20352 }
20353}
20354
20355// FastEmit functions for ISD::ROTL.
20356
20357Register fastEmit_ISD_ROTL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20358 if (RetVT.SimpleTy != MVT::i8)
20359 return Register();
20360 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20361 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20362 }
20363 if ((!Subtarget->hasNDD())) {
20364 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20365 }
20366 return Register();
20367}
20368
20369Register fastEmit_ISD_ROTL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20370 if (RetVT.SimpleTy != MVT::i16)
20371 return Register();
20372 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20373 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20374 }
20375 if ((!Subtarget->hasNDD())) {
20376 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20377 }
20378 return Register();
20379}
20380
20381Register fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20382 if (RetVT.SimpleTy != MVT::i32)
20383 return Register();
20384 if ((Subtarget->hasFastSHLDRotate())) {
20385 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20386 }
20387 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20388 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20389 }
20390 if ((!Subtarget->hasNDD())) {
20391 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20392 }
20393 return Register();
20394}
20395
20396Register fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20397 if (RetVT.SimpleTy != MVT::i64)
20398 return Register();
20399 if ((Subtarget->hasFastSHLDRotate())) {
20400 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20401 }
20402 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20403 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20404 }
20405 if ((!Subtarget->hasNDD())) {
20406 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20407 }
20408 return Register();
20409}
20410
20411Register fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20412 switch (VT.SimpleTy) {
20413 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_ri(RetVT, Op0, imm1);
20414 case MVT::i16: return fastEmit_ISD_ROTL_MVT_i16_ri(RetVT, Op0, imm1);
20415 case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, imm1);
20416 case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, imm1);
20417 default: return Register();
20418 }
20419}
20420
20421// FastEmit functions for ISD::ROTR.
20422
20423Register fastEmit_ISD_ROTR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20424 if (RetVT.SimpleTy != MVT::i8)
20425 return Register();
20426 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20427 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20428 }
20429 if ((!Subtarget->hasNDD())) {
20430 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20431 }
20432 return Register();
20433}
20434
20435Register fastEmit_ISD_ROTR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20436 if (RetVT.SimpleTy != MVT::i16)
20437 return Register();
20438 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20439 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20440 }
20441 if ((!Subtarget->hasNDD())) {
20442 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20443 }
20444 return Register();
20445}
20446
20447Register fastEmit_ISD_ROTR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20448 if (RetVT.SimpleTy != MVT::i32)
20449 return Register();
20450 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
20451 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri_EVEX, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20452 }
20453 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
20454 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20455 }
20456 if ((Subtarget->hasFastSHLDRotate())) {
20457 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20458 }
20459 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20460 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20461 }
20462 if ((!Subtarget->hasNDD())) {
20463 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20464 }
20465 return Register();
20466}
20467
20468Register fastEmit_ISD_ROTR_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20469 if (RetVT.SimpleTy != MVT::i64)
20470 return Register();
20471 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
20472 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri_EVEX, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20473 }
20474 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
20475 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20476 }
20477 if ((Subtarget->hasFastSHLDRotate())) {
20478 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20479 }
20480 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20481 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20482 }
20483 if ((!Subtarget->hasNDD())) {
20484 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20485 }
20486 return Register();
20487}
20488
20489Register fastEmit_ISD_ROTR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20490 switch (VT.SimpleTy) {
20491 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_ri(RetVT, Op0, imm1);
20492 case MVT::i16: return fastEmit_ISD_ROTR_MVT_i16_ri(RetVT, Op0, imm1);
20493 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri(RetVT, Op0, imm1);
20494 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_ri(RetVT, Op0, imm1);
20495 default: return Register();
20496 }
20497}
20498
20499// FastEmit functions for ISD::SHL.
20500
20501Register fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20502 if (RetVT.SimpleTy != MVT::i8)
20503 return Register();
20504 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20505 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20506 }
20507 if ((!Subtarget->hasNDD())) {
20508 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20509 }
20510 return Register();
20511}
20512
20513Register fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20514 if (RetVT.SimpleTy != MVT::i16)
20515 return Register();
20516 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20517 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20518 }
20519 if ((!Subtarget->hasNDD())) {
20520 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20521 }
20522 return Register();
20523}
20524
20525Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20526 if (RetVT.SimpleTy != MVT::i32)
20527 return Register();
20528 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20529 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20530 }
20531 if ((!Subtarget->hasNDD())) {
20532 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20533 }
20534 return Register();
20535}
20536
20537Register fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20538 if (RetVT.SimpleTy != MVT::i64)
20539 return Register();
20540 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20541 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20542 }
20543 if ((!Subtarget->hasNDD())) {
20544 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20545 }
20546 return Register();
20547}
20548
20549Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20550 switch (VT.SimpleTy) {
20551 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, imm1);
20552 case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, imm1);
20553 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
20554 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, imm1);
20555 default: return Register();
20556 }
20557}
20558
20559// FastEmit functions for ISD::SRA.
20560
20561Register fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20562 if (RetVT.SimpleTy != MVT::i8)
20563 return Register();
20564 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20565 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20566 }
20567 if ((!Subtarget->hasNDD())) {
20568 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20569 }
20570 return Register();
20571}
20572
20573Register fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20574 if (RetVT.SimpleTy != MVT::i16)
20575 return Register();
20576 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20577 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20578 }
20579 if ((!Subtarget->hasNDD())) {
20580 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20581 }
20582 return Register();
20583}
20584
20585Register fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20586 if (RetVT.SimpleTy != MVT::i32)
20587 return Register();
20588 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20589 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20590 }
20591 if ((!Subtarget->hasNDD())) {
20592 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20593 }
20594 return Register();
20595}
20596
20597Register fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20598 if (RetVT.SimpleTy != MVT::i64)
20599 return Register();
20600 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20601 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20602 }
20603 if ((!Subtarget->hasNDD())) {
20604 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20605 }
20606 return Register();
20607}
20608
20609Register fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20610 switch (VT.SimpleTy) {
20611 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, imm1);
20612 case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, imm1);
20613 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, imm1);
20614 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, imm1);
20615 default: return Register();
20616 }
20617}
20618
20619// FastEmit functions for ISD::SRL.
20620
20621Register fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20622 if (RetVT.SimpleTy != MVT::i8)
20623 return Register();
20624 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20625 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20626 }
20627 if ((!Subtarget->hasNDD())) {
20628 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20629 }
20630 return Register();
20631}
20632
20633Register fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20634 if (RetVT.SimpleTy != MVT::i16)
20635 return Register();
20636 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20637 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20638 }
20639 if ((!Subtarget->hasNDD())) {
20640 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20641 }
20642 return Register();
20643}
20644
20645Register fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20646 if (RetVT.SimpleTy != MVT::i32)
20647 return Register();
20648 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20649 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20650 }
20651 if ((!Subtarget->hasNDD())) {
20652 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20653 }
20654 return Register();
20655}
20656
20657Register fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20658 if (RetVT.SimpleTy != MVT::i64)
20659 return Register();
20660 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20661 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20662 }
20663 if ((!Subtarget->hasNDD())) {
20664 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20665 }
20666 return Register();
20667}
20668
20669Register fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20670 switch (VT.SimpleTy) {
20671 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, imm1);
20672 case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, imm1);
20673 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, imm1);
20674 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, imm1);
20675 default: return Register();
20676 }
20677}
20678
20679// FastEmit functions for ISD::SUB.
20680
20681Register fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20682 if (RetVT.SimpleTy != MVT::i8)
20683 return Register();
20684 if ((Subtarget->hasNDD())) {
20685 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20686 }
20687 if ((!Subtarget->hasNDD())) {
20688 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20689 }
20690 return Register();
20691}
20692
20693Register fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20694 if (RetVT.SimpleTy != MVT::i16)
20695 return Register();
20696 if ((Subtarget->hasNDD())) {
20697 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20698 }
20699 if ((!Subtarget->hasNDD())) {
20700 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20701 }
20702 return Register();
20703}
20704
20705Register fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20706 if (RetVT.SimpleTy != MVT::i32)
20707 return Register();
20708 if ((Subtarget->hasNDD())) {
20709 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20710 }
20711 if ((!Subtarget->hasNDD())) {
20712 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20713 }
20714 return Register();
20715}
20716
20717Register fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20718 switch (VT.SimpleTy) {
20719 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, imm1);
20720 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, imm1);
20721 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, imm1);
20722 default: return Register();
20723 }
20724}
20725
20726// FastEmit functions for ISD::XOR.
20727
20728Register fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20729 if (RetVT.SimpleTy != MVT::i8)
20730 return Register();
20731 if ((Subtarget->hasNDD())) {
20732 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20733 }
20734 if ((!Subtarget->hasNDD())) {
20735 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20736 }
20737 return Register();
20738}
20739
20740Register fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20741 if (RetVT.SimpleTy != MVT::i16)
20742 return Register();
20743 if ((Subtarget->hasNDD())) {
20744 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20745 }
20746 if ((!Subtarget->hasNDD())) {
20747 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20748 }
20749 return Register();
20750}
20751
20752Register fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20753 if (RetVT.SimpleTy != MVT::i32)
20754 return Register();
20755 if ((Subtarget->hasNDD())) {
20756 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20757 }
20758 if ((!Subtarget->hasNDD())) {
20759 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20760 }
20761 return Register();
20762}
20763
20764Register fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20765 switch (VT.SimpleTy) {
20766 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, imm1);
20767 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, imm1);
20768 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, imm1);
20769 default: return Register();
20770 }
20771}
20772
20773// Top-level FastEmit function.
20774
20775Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
20776 if (VT == MVT::i64 && Predicate_i64immSExt32(Imm: imm1))
20777 if (Register Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, imm1))
20778 return Reg;
20779
20780 switch (Opcode) {
20781 case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, imm1);
20782 case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, imm1);
20783 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
20784 case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, imm1);
20785 case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, imm1);
20786 case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, imm1);
20787 case ISD::ROTR: return fastEmit_ISD_ROTR_ri(VT, RetVT, Op0, imm1);
20788 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
20789 case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, imm1);
20790 case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, imm1);
20791 case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, imm1);
20792 case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, imm1);
20793 default: return Register();
20794 }
20795}
20796
20797// FastEmit functions for ISD::ADD.
20798
20799Register fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
20800 if (RetVT.SimpleTy != MVT::i64)
20801 return Register();
20802 if ((Subtarget->hasNDD())) {
20803 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20804 }
20805 if ((!Subtarget->hasNDD())) {
20806 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20807 }
20808 return Register();
20809}
20810
20811Register fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20812 switch (VT.SimpleTy) {
20813 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
20814 default: return Register();
20815 }
20816}
20817
20818// FastEmit functions for ISD::AND.
20819
20820Register fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
20821 if (RetVT.SimpleTy != MVT::i64)
20822 return Register();
20823 if ((Subtarget->hasNDD())) {
20824 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20825 }
20826 if ((!Subtarget->hasNDD())) {
20827 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20828 }
20829 return Register();
20830}
20831
20832Register fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20833 switch (VT.SimpleTy) {
20834 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
20835 default: return Register();
20836 }
20837}
20838
20839// FastEmit functions for ISD::MUL.
20840
20841Register fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
20842 if (RetVT.SimpleTy != MVT::i64)
20843 return Register();
20844 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL64rri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20845}
20846
20847Register fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20848 switch (VT.SimpleTy) {
20849 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
20850 default: return Register();
20851 }
20852}
20853
20854// FastEmit functions for ISD::OR.
20855
20856Register fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
20857 if (RetVT.SimpleTy != MVT::i64)
20858 return Register();
20859 if ((Subtarget->hasNDD())) {
20860 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20861 }
20862 if ((!Subtarget->hasNDD())) {
20863 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20864 }
20865 return Register();
20866}
20867
20868Register fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20869 switch (VT.SimpleTy) {
20870 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
20871 default: return Register();
20872 }
20873}
20874
20875// FastEmit functions for ISD::SUB.
20876
20877Register fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
20878 if (RetVT.SimpleTy != MVT::i64)
20879 return Register();
20880 if ((Subtarget->hasNDD())) {
20881 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20882 }
20883 if ((!Subtarget->hasNDD())) {
20884 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20885 }
20886 return Register();
20887}
20888
20889Register fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20890 switch (VT.SimpleTy) {
20891 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
20892 default: return Register();
20893 }
20894}
20895
20896// FastEmit functions for ISD::XOR.
20897
20898Register fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
20899 if (RetVT.SimpleTy != MVT::i64)
20900 return Register();
20901 if ((Subtarget->hasNDD())) {
20902 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20903 }
20904 if ((!Subtarget->hasNDD())) {
20905 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20906 }
20907 return Register();
20908}
20909
20910Register fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20911 switch (VT.SimpleTy) {
20912 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
20913 default: return Register();
20914 }
20915}
20916
20917// Top-level FastEmit function.
20918
20919Register fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
20920 switch (Opcode) {
20921 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
20922 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
20923 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
20924 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
20925 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
20926 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
20927 default: return Register();
20928 }
20929}
20930
20931// FastEmit functions for ISD::Constant.
20932
20933Register fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) {
20934 if (RetVT.SimpleTy != MVT::i8)
20935 return Register();
20936 return fastEmitInst_i(MachineInstOpcode: X86::MOV8ri, RC: &X86::GR8RegClass, Imm: imm0);
20937}
20938
20939Register fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) {
20940 if (RetVT.SimpleTy != MVT::i16)
20941 return Register();
20942 return fastEmitInst_i(MachineInstOpcode: X86::MOV16ri, RC: &X86::GR16RegClass, Imm: imm0);
20943}
20944
20945Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
20946 if (RetVT.SimpleTy != MVT::i32)
20947 return Register();
20948 return fastEmitInst_i(MachineInstOpcode: X86::MOV32ri, RC: &X86::GR32RegClass, Imm: imm0);
20949}
20950
20951Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
20952 if (RetVT.SimpleTy != MVT::i64)
20953 return Register();
20954 return fastEmitInst_i(MachineInstOpcode: X86::MOV64ri, RC: &X86::GR64RegClass, Imm: imm0);
20955}
20956
20957Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
20958 switch (VT.SimpleTy) {
20959 case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0);
20960 case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0);
20961 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
20962 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
20963 default: return Register();
20964 }
20965}
20966
20967// Top-level FastEmit function.
20968
20969Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
20970 switch (Opcode) {
20971 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
20972 default: return Register();
20973 }
20974}
20975
20976