1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the X86 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_AndMask64(int64_t Imm) {
12
13 return isMask_64(Value: Imm) && !isUInt<32>(x: Imm);
14
15}
16static bool Predicate_BTRMask64(int64_t Imm) {
17
18 return !isUInt<32>(x: Imm) && !isInt<32>(x: Imm) && isPowerOf2_64(Value: ~Imm);
19
20}
21static bool Predicate_BTCBTSMask64(int64_t Imm) {
22
23 return !isInt<32>(x: Imm) && isPowerOf2_64(Value: Imm);
24
25}
26static bool Predicate_i64immSExt32(int64_t Imm) {
27 return isInt<32>(x: Imm);
28}
29
30
31// FastEmit functions for ISD::ABS.
32
33Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
34 if (RetVT.SimpleTy != MVT::v16i8)
35 return Register();
36 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
37 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ128rr, RC: &X86::VR128XRegClass, Op0);
38 }
39 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
40 return fastEmitInst_r(MachineInstOpcode: X86::PABSBrr, RC: &X86::VR128RegClass, Op0);
41 }
42 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
43 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBrr, RC: &X86::VR128RegClass, Op0);
44 }
45 return Register();
46}
47
48Register fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, Register Op0) {
49 if (RetVT.SimpleTy != MVT::v32i8)
50 return Register();
51 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
52 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ256rr, RC: &X86::VR256XRegClass, Op0);
53 }
54 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
55 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBYrr, RC: &X86::VR256RegClass, Op0);
56 }
57 return Register();
58}
59
60Register fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, Register Op0) {
61 if (RetVT.SimpleTy != MVT::v64i8)
62 return Register();
63 if ((Subtarget->hasBWI())) {
64 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZrr, RC: &X86::VR512RegClass, Op0);
65 }
66 return Register();
67}
68
69Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
70 if (RetVT.SimpleTy != MVT::v8i16)
71 return Register();
72 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
73 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ128rr, RC: &X86::VR128XRegClass, Op0);
74 }
75 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
76 return fastEmitInst_r(MachineInstOpcode: X86::PABSWrr, RC: &X86::VR128RegClass, Op0);
77 }
78 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
79 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWrr, RC: &X86::VR128RegClass, Op0);
80 }
81 return Register();
82}
83
84Register fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, Register Op0) {
85 if (RetVT.SimpleTy != MVT::v16i16)
86 return Register();
87 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
88 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ256rr, RC: &X86::VR256XRegClass, Op0);
89 }
90 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
91 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWYrr, RC: &X86::VR256RegClass, Op0);
92 }
93 return Register();
94}
95
96Register fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, Register Op0) {
97 if (RetVT.SimpleTy != MVT::v32i16)
98 return Register();
99 if ((Subtarget->hasBWI())) {
100 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZrr, RC: &X86::VR512RegClass, Op0);
101 }
102 return Register();
103}
104
105Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
106 if (RetVT.SimpleTy != MVT::v4i32)
107 return Register();
108 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
109 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ128rr, RC: &X86::VR128XRegClass, Op0);
110 }
111 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
112 return fastEmitInst_r(MachineInstOpcode: X86::PABSDrr, RC: &X86::VR128RegClass, Op0);
113 }
114 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
115 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDrr, RC: &X86::VR128RegClass, Op0);
116 }
117 return Register();
118}
119
120Register fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, Register Op0) {
121 if (RetVT.SimpleTy != MVT::v8i32)
122 return Register();
123 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
124 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ256rr, RC: &X86::VR256XRegClass, Op0);
125 }
126 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
127 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDYrr, RC: &X86::VR256RegClass, Op0);
128 }
129 return Register();
130}
131
132Register fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, Register Op0) {
133 if (RetVT.SimpleTy != MVT::v16i32)
134 return Register();
135 if ((Subtarget->hasAVX512())) {
136 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZrr, RC: &X86::VR512RegClass, Op0);
137 }
138 return Register();
139}
140
141Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
142 if (RetVT.SimpleTy != MVT::v2i64)
143 return Register();
144 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
145 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ128rr, RC: &X86::VR128XRegClass, Op0);
146 }
147 return Register();
148}
149
150Register fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, Register Op0) {
151 if (RetVT.SimpleTy != MVT::v4i64)
152 return Register();
153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
154 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ256rr, RC: &X86::VR256XRegClass, Op0);
155 }
156 return Register();
157}
158
159Register fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, Register Op0) {
160 if (RetVT.SimpleTy != MVT::v8i64)
161 return Register();
162 if ((Subtarget->hasAVX512())) {
163 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZrr, RC: &X86::VR512RegClass, Op0);
164 }
165 return Register();
166}
167
168Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
169 switch (VT.SimpleTy) {
170 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
171 case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0);
172 case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0);
173 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
174 case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0);
175 case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0);
176 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
177 case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0);
178 case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0);
179 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
180 case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0);
181 case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0);
182 default: return Register();
183 }
184}
185
186// FastEmit functions for ISD::ANY_EXTEND.
187
188Register fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
189 if (RetVT.SimpleTy != MVT::i32)
190 return Register();
191 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
192}
193
194Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
195 switch (VT.SimpleTy) {
196 case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0);
197 default: return Register();
198 }
199}
200
201// FastEmit functions for ISD::BITCAST.
202
203Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
204 if (RetVT.SimpleTy != MVT::f32)
205 return Register();
206 if ((Subtarget->hasAVX512())) {
207 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSZrr, RC: &X86::FR32XRegClass, Op0);
208 }
209 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
210 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
211 }
212 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
213 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
214 }
215 return Register();
216}
217
218Register fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Register Op0) {
219 if ((Subtarget->hasAVX512())) {
220 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDZrr, RC: &X86::FR64XRegClass, Op0);
221 }
222 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
223 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toSDrr, RC: &X86::FR64RegClass, Op0);
224 }
225 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
226 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDrr, RC: &X86::FR64RegClass, Op0);
227 }
228 return Register();
229}
230
231Register fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Register Op0) {
232 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
233 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64to64rr, RC: &X86::VR64RegClass, Op0);
234 }
235 return Register();
236}
237
238Register fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, Register Op0) {
239switch (RetVT.SimpleTy) {
240 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0);
241 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0);
242 default: return Register();
243}
244}
245
246Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
247 if (RetVT.SimpleTy != MVT::i32)
248 return Register();
249 if ((Subtarget->hasAVX512())) {
250 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIZrr, RC: &X86::GR32RegClass, Op0);
251 }
252 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
253 return fastEmitInst_r(MachineInstOpcode: X86::MOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
254 }
255 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
256 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
257 }
258 return Register();
259}
260
261Register fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Register Op0) {
262 if ((Subtarget->hasAVX512())) {
263 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64Zrr, RC: &X86::GR64RegClass, Op0);
264 }
265 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
266 return fastEmitInst_r(MachineInstOpcode: X86::MOVSDto64rr, RC: &X86::GR64RegClass, Op0);
267 }
268 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
269 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64rr, RC: &X86::GR64RegClass, Op0);
270 }
271 return Register();
272}
273
274Register fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Register Op0) {
275 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
276 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVFR642Qrr, RC: &X86::VR64RegClass, Op0);
277 }
278 return Register();
279}
280
281Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
282switch (RetVT.SimpleTy) {
283 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0);
284 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0);
285 default: return Register();
286}
287}
288
289Register fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Register Op0) {
290 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
291 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64from64rr, RC: &X86::GR64RegClass, Op0);
292 }
293 return Register();
294}
295
296Register fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Register Op0) {
297 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
298 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2FR64rr, RC: &X86::FR64RegClass, Op0);
299 }
300 return Register();
301}
302
303Register fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, Register Op0) {
304switch (RetVT.SimpleTy) {
305 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0);
306 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0);
307 default: return Register();
308}
309}
310
311Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
312 switch (VT.SimpleTy) {
313 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
314 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0);
315 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
316 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
317 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0);
318 default: return Register();
319 }
320}
321
322// FastEmit functions for ISD::BRIND.
323
324Register fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, Register Op0) {
325 if (RetVT.SimpleTy != MVT::isVoid)
326 return Register();
327 if ((!Subtarget->is64Bit())) {
328 return fastEmitInst_r(MachineInstOpcode: X86::JMP16r, RC: &X86::GR16RegClass, Op0);
329 }
330 return Register();
331}
332
333Register fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
334 if (RetVT.SimpleTy != MVT::isVoid)
335 return Register();
336 if ((!Subtarget->is64Bit())) {
337 return fastEmitInst_r(MachineInstOpcode: X86::JMP32r, RC: &X86::GR32RegClass, Op0);
338 }
339 return Register();
340}
341
342Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
343 if (RetVT.SimpleTy != MVT::isVoid)
344 return Register();
345 if ((Subtarget->is64Bit())) {
346 return fastEmitInst_r(MachineInstOpcode: X86::JMP64r, RC: &X86::GR64RegClass, Op0);
347 }
348 return Register();
349}
350
351Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
352 switch (VT.SimpleTy) {
353 case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0);
354 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
355 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
356 default: return Register();
357 }
358}
359
360// FastEmit functions for ISD::BSWAP.
361
362Register fastEmit_ISD_BSWAP_MVT_i16_r(MVT RetVT, Register Op0) {
363 if (RetVT.SimpleTy != MVT::i16)
364 return Register();
365 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
366 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE16rr, RC: &X86::GR16RegClass, Op0);
367 }
368 return Register();
369}
370
371Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
372 if (RetVT.SimpleTy != MVT::i32)
373 return Register();
374 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
375 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE32rr, RC: &X86::GR32RegClass, Op0);
376 }
377 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
378 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP32r, RC: &X86::GR32RegClass, Op0);
379 }
380 return Register();
381}
382
383Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
384 if (RetVT.SimpleTy != MVT::i64)
385 return Register();
386 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
387 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE64rr, RC: &X86::GR64RegClass, Op0);
388 }
389 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
390 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP64r, RC: &X86::GR64RegClass, Op0);
391 }
392 return Register();
393}
394
395Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
396 switch (VT.SimpleTy) {
397 case MVT::i16: return fastEmit_ISD_BSWAP_MVT_i16_r(RetVT, Op0);
398 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
399 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
400 default: return Register();
401 }
402}
403
404// FastEmit functions for ISD::CTLZ.
405
406Register fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, Register Op0) {
407 if (RetVT.SimpleTy != MVT::i16)
408 return Register();
409 if ((Subtarget->hasLZCNT())) {
410 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT16rr, RC: &X86::GR16RegClass, Op0);
411 }
412 return Register();
413}
414
415Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
416 if (RetVT.SimpleTy != MVT::i32)
417 return Register();
418 if ((Subtarget->hasLZCNT())) {
419 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT32rr, RC: &X86::GR32RegClass, Op0);
420 }
421 return Register();
422}
423
424Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
425 if (RetVT.SimpleTy != MVT::i64)
426 return Register();
427 if ((Subtarget->hasLZCNT())) {
428 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT64rr, RC: &X86::GR64RegClass, Op0);
429 }
430 return Register();
431}
432
433Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
434 if (RetVT.SimpleTy != MVT::v4i32)
435 return Register();
436 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
437 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
438 }
439 return Register();
440}
441
442Register fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, Register Op0) {
443 if (RetVT.SimpleTy != MVT::v8i32)
444 return Register();
445 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
446 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
447 }
448 return Register();
449}
450
451Register fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, Register Op0) {
452 if (RetVT.SimpleTy != MVT::v16i32)
453 return Register();
454 if ((Subtarget->hasCDI())) {
455 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZrr, RC: &X86::VR512RegClass, Op0);
456 }
457 return Register();
458}
459
460Register fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, Register Op0) {
461 if (RetVT.SimpleTy != MVT::v2i64)
462 return Register();
463 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
464 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
465 }
466 return Register();
467}
468
469Register fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, Register Op0) {
470 if (RetVT.SimpleTy != MVT::v4i64)
471 return Register();
472 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
473 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
474 }
475 return Register();
476}
477
478Register fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, Register Op0) {
479 if (RetVT.SimpleTy != MVT::v8i64)
480 return Register();
481 if ((Subtarget->hasCDI())) {
482 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZrr, RC: &X86::VR512RegClass, Op0);
483 }
484 return Register();
485}
486
487Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
488 switch (VT.SimpleTy) {
489 case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0);
490 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
491 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
492 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
493 case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0);
494 case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0);
495 case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0);
496 case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0);
497 case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0);
498 default: return Register();
499 }
500}
501
502// FastEmit functions for ISD::CTPOP.
503
504Register fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, Register Op0) {
505 if (RetVT.SimpleTy != MVT::i16)
506 return Register();
507 if ((Subtarget->hasPOPCNT())) {
508 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT16rr, RC: &X86::GR16RegClass, Op0);
509 }
510 return Register();
511}
512
513Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
514 if (RetVT.SimpleTy != MVT::i32)
515 return Register();
516 if ((Subtarget->hasPOPCNT())) {
517 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT32rr, RC: &X86::GR32RegClass, Op0);
518 }
519 return Register();
520}
521
522Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
523 if (RetVT.SimpleTy != MVT::i64)
524 return Register();
525 if ((Subtarget->hasPOPCNT())) {
526 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT64rr, RC: &X86::GR64RegClass, Op0);
527 }
528 return Register();
529}
530
531Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
532 if (RetVT.SimpleTy != MVT::v16i8)
533 return Register();
534 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
535 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ128rr, RC: &X86::VR128XRegClass, Op0);
536 }
537 return Register();
538}
539
540Register fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, Register Op0) {
541 if (RetVT.SimpleTy != MVT::v32i8)
542 return Register();
543 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
544 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ256rr, RC: &X86::VR256XRegClass, Op0);
545 }
546 return Register();
547}
548
549Register fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, Register Op0) {
550 if (RetVT.SimpleTy != MVT::v64i8)
551 return Register();
552 if ((Subtarget->hasBITALG())) {
553 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZrr, RC: &X86::VR512RegClass, Op0);
554 }
555 return Register();
556}
557
558Register fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, Register Op0) {
559 if (RetVT.SimpleTy != MVT::v8i16)
560 return Register();
561 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
562 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ128rr, RC: &X86::VR128XRegClass, Op0);
563 }
564 return Register();
565}
566
567Register fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, Register Op0) {
568 if (RetVT.SimpleTy != MVT::v16i16)
569 return Register();
570 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
571 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ256rr, RC: &X86::VR256XRegClass, Op0);
572 }
573 return Register();
574}
575
576Register fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, Register Op0) {
577 if (RetVT.SimpleTy != MVT::v32i16)
578 return Register();
579 if ((Subtarget->hasBITALG())) {
580 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZrr, RC: &X86::VR512RegClass, Op0);
581 }
582 return Register();
583}
584
585Register fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, Register Op0) {
586 if (RetVT.SimpleTy != MVT::v4i32)
587 return Register();
588 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
589 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
590 }
591 return Register();
592}
593
594Register fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, Register Op0) {
595 if (RetVT.SimpleTy != MVT::v8i32)
596 return Register();
597 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
598 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
599 }
600 return Register();
601}
602
603Register fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, Register Op0) {
604 if (RetVT.SimpleTy != MVT::v16i32)
605 return Register();
606 if ((Subtarget->hasVPOPCNTDQ())) {
607 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZrr, RC: &X86::VR512RegClass, Op0);
608 }
609 return Register();
610}
611
612Register fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, Register Op0) {
613 if (RetVT.SimpleTy != MVT::v2i64)
614 return Register();
615 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
616 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
617 }
618 return Register();
619}
620
621Register fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, Register Op0) {
622 if (RetVT.SimpleTy != MVT::v4i64)
623 return Register();
624 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
625 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
626 }
627 return Register();
628}
629
630Register fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, Register Op0) {
631 if (RetVT.SimpleTy != MVT::v8i64)
632 return Register();
633 if ((Subtarget->hasVPOPCNTDQ())) {
634 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZrr, RC: &X86::VR512RegClass, Op0);
635 }
636 return Register();
637}
638
639Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
640 switch (VT.SimpleTy) {
641 case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0);
642 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
643 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
644 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
645 case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0);
646 case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0);
647 case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0);
648 case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0);
649 case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0);
650 case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0);
651 case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0);
652 case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0);
653 case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0);
654 case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0);
655 case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0);
656 default: return Register();
657 }
658}
659
660// FastEmit functions for ISD::CTTZ.
661
662Register fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, Register Op0) {
663 if (RetVT.SimpleTy != MVT::i16)
664 return Register();
665 if ((Subtarget->hasBMI())) {
666 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT16rr, RC: &X86::GR16RegClass, Op0);
667 }
668 return Register();
669}
670
671Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
672 if (RetVT.SimpleTy != MVT::i32)
673 return Register();
674 if ((Subtarget->hasBMI())) {
675 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT32rr, RC: &X86::GR32RegClass, Op0);
676 }
677 return Register();
678}
679
680Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
681 if (RetVT.SimpleTy != MVT::i64)
682 return Register();
683 if ((Subtarget->hasBMI())) {
684 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT64rr, RC: &X86::GR64RegClass, Op0);
685 }
686 return Register();
687}
688
689Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
690 switch (VT.SimpleTy) {
691 case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0);
692 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
693 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
694 default: return Register();
695 }
696}
697
698// FastEmit functions for ISD::FABS.
699
700Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
701 if (RetVT.SimpleTy != MVT::f32)
702 return Register();
703 if ((!Subtarget->hasSSE1())) {
704 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp32, RC: &X86::RFP32RegClass, Op0);
705 }
706 return Register();
707}
708
709Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
710 if (RetVT.SimpleTy != MVT::f64)
711 return Register();
712 if ((!Subtarget->hasSSE2())) {
713 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp64, RC: &X86::RFP64RegClass, Op0);
714 }
715 return Register();
716}
717
718Register fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, Register Op0) {
719 if (RetVT.SimpleTy != MVT::f80)
720 return Register();
721 if ((Subtarget->hasX87())) {
722 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp80, RC: &X86::RFP80RegClass, Op0);
723 }
724 return Register();
725}
726
727Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
728 switch (VT.SimpleTy) {
729 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
730 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
731 case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0);
732 default: return Register();
733 }
734}
735
736// FastEmit functions for ISD::FNEG.
737
738Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
739 if (RetVT.SimpleTy != MVT::f32)
740 return Register();
741 if ((!Subtarget->hasSSE1())) {
742 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp32, RC: &X86::RFP32RegClass, Op0);
743 }
744 return Register();
745}
746
747Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
748 if (RetVT.SimpleTy != MVT::f64)
749 return Register();
750 if ((!Subtarget->hasSSE2())) {
751 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp64, RC: &X86::RFP64RegClass, Op0);
752 }
753 return Register();
754}
755
756Register fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, Register Op0) {
757 if (RetVT.SimpleTy != MVT::f80)
758 return Register();
759 if ((Subtarget->hasX87())) {
760 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp80, RC: &X86::RFP80RegClass, Op0);
761 }
762 return Register();
763}
764
765Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
766 switch (VT.SimpleTy) {
767 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
768 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
769 case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0);
770 default: return Register();
771 }
772}
773
774// FastEmit functions for ISD::FP_EXTEND.
775
776Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
777 if (RetVT.SimpleTy != MVT::f64)
778 return Register();
779 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
780 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
781 }
782 return Register();
783}
784
785Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Register Op0) {
786 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
787 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
788 }
789 return Register();
790}
791
792Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Register Op0) {
793 if ((Subtarget->hasFP16())) {
794 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
795 }
796 return Register();
797}
798
799Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(MVT RetVT, Register Op0) {
800switch (RetVT.SimpleTy) {
801 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
802 case MVT::v8f64: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
803 default: return Register();
804}
805}
806
807Register fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(MVT RetVT, Register Op0) {
808 if (RetVT.SimpleTy != MVT::v16f32)
809 return Register();
810 if ((Subtarget->hasFP16())) {
811 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
812 }
813 return Register();
814}
815
816Register fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, Register Op0) {
817 if (RetVT.SimpleTy != MVT::v4f64)
818 return Register();
819 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
820 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
821 }
822 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
823 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
824 }
825 return Register();
826}
827
828Register fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, Register Op0) {
829 if (RetVT.SimpleTy != MVT::v8f64)
830 return Register();
831 if ((Subtarget->hasAVX512())) {
832 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
833 }
834 return Register();
835}
836
837Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
838 switch (VT.SimpleTy) {
839 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
840 case MVT::v8f16: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
841 case MVT::v16f16: return fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
842 case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
843 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
844 default: return Register();
845 }
846}
847
848// FastEmit functions for ISD::FP_ROUND.
849
850Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
851 if (RetVT.SimpleTy != MVT::f32)
852 return Register();
853 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
854 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
855 }
856 return Register();
857}
858
859Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
860 switch (VT.SimpleTy) {
861 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
862 default: return Register();
863 }
864}
865
866// FastEmit functions for ISD::FP_TO_SINT.
867
868Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
869 if ((Subtarget->hasFP16())) {
870 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
871 }
872 return Register();
873}
874
875Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
876 if ((Subtarget->hasFP16())) {
877 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
878 }
879 return Register();
880}
881
882Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
883switch (RetVT.SimpleTy) {
884 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
885 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
886 default: return Register();
887}
888}
889
890Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
891 if ((Subtarget->hasAVX512())) {
892 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
893 }
894 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
895 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
896 }
897 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
898 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
899 }
900 return Register();
901}
902
903Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
904 if ((Subtarget->hasAVX512())) {
905 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
906 }
907 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
908 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
909 }
910 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
911 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
912 }
913 return Register();
914}
915
916Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
917switch (RetVT.SimpleTy) {
918 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
919 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
920 default: return Register();
921}
922}
923
924Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
925 if ((Subtarget->hasAVX512())) {
926 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
927 }
928 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
929 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
930 }
931 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
932 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
933 }
934 return Register();
935}
936
937Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
938 if ((Subtarget->hasAVX512())) {
939 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
940 }
941 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
942 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
943 }
944 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
945 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
946 }
947 return Register();
948}
949
950Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
951switch (RetVT.SimpleTy) {
952 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
953 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
954 default: return Register();
955}
956}
957
958Register fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
959 if (RetVT.SimpleTy != MVT::v4i32)
960 return Register();
961 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
963 }
964 return Register();
965}
966
967Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
968 switch (VT.SimpleTy) {
969 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
970 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
971 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
972 case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
973 default: return Register();
974 }
975}
976
977// FastEmit functions for ISD::FP_TO_UINT.
978
979Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
980 if ((Subtarget->hasFP16())) {
981 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
982 }
983 return Register();
984}
985
986Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
987 if ((Subtarget->hasFP16())) {
988 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
989 }
990 return Register();
991}
992
993Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
994switch (RetVT.SimpleTy) {
995 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
996 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
997 default: return Register();
998}
999}
1000
1001Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
1002 if ((Subtarget->hasAVX512())) {
1003 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
1004 }
1005 return Register();
1006}
1007
1008Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
1009 if ((Subtarget->hasAVX512())) {
1010 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1011 }
1012 return Register();
1013}
1014
1015Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
1016switch (RetVT.SimpleTy) {
1017 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
1018 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
1019 default: return Register();
1020}
1021}
1022
1023Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
1024 if ((Subtarget->hasAVX512())) {
1025 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
1026 }
1027 return Register();
1028}
1029
1030Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
1031 if ((Subtarget->hasAVX512())) {
1032 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1033 }
1034 return Register();
1035}
1036
1037Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
1038switch (RetVT.SimpleTy) {
1039 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
1040 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
1041 default: return Register();
1042}
1043}
1044
1045Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
1046 switch (VT.SimpleTy) {
1047 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
1048 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
1049 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
1050 default: return Register();
1051 }
1052}
1053
1054// FastEmit functions for ISD::FSQRT.
1055
1056Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
1057 if (RetVT.SimpleTy != MVT::f32)
1058 return Register();
1059 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1060 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
1061 }
1062 if ((!Subtarget->hasSSE1())) {
1063 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
1064 }
1065 return Register();
1066}
1067
1068Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
1069 if (RetVT.SimpleTy != MVT::f64)
1070 return Register();
1071 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1072 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
1073 }
1074 if ((!Subtarget->hasSSE2())) {
1075 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
1076 }
1077 return Register();
1078}
1079
1080Register fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, Register Op0) {
1081 if (RetVT.SimpleTy != MVT::f80)
1082 return Register();
1083 if ((Subtarget->hasX87())) {
1084 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
1085 }
1086 return Register();
1087}
1088
1089Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1090 if (RetVT.SimpleTy != MVT::v8f16)
1091 return Register();
1092 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1093 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
1094 }
1095 return Register();
1096}
1097
1098Register fastEmit_ISD_FSQRT_MVT_v16f16_r(MVT RetVT, Register Op0) {
1099 if (RetVT.SimpleTy != MVT::v16f16)
1100 return Register();
1101 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1102 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
1103 }
1104 return Register();
1105}
1106
1107Register fastEmit_ISD_FSQRT_MVT_v32f16_r(MVT RetVT, Register Op0) {
1108 if (RetVT.SimpleTy != MVT::v32f16)
1109 return Register();
1110 if ((Subtarget->hasFP16())) {
1111 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
1112 }
1113 return Register();
1114}
1115
1116Register fastEmit_ISD_FSQRT_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1117 if (RetVT.SimpleTy != MVT::v8bf16)
1118 return Register();
1119 if ((Subtarget->hasAVX10_2())) {
1120 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
1121 }
1122 return Register();
1123}
1124
1125Register fastEmit_ISD_FSQRT_MVT_v16bf16_r(MVT RetVT, Register Op0) {
1126 if (RetVT.SimpleTy != MVT::v16bf16)
1127 return Register();
1128 if ((Subtarget->hasAVX10_2())) {
1129 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
1130 }
1131 return Register();
1132}
1133
1134Register fastEmit_ISD_FSQRT_MVT_v32bf16_r(MVT RetVT, Register Op0) {
1135 if (RetVT.SimpleTy != MVT::v32bf16)
1136 return Register();
1137 if ((Subtarget->hasAVX10_2())) {
1138 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
1139 }
1140 return Register();
1141}
1142
1143Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1144 if (RetVT.SimpleTy != MVT::v4f32)
1145 return Register();
1146 if ((Subtarget->hasVLX())) {
1147 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
1148 }
1149 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1150 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
1151 }
1152 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1153 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
1154 }
1155 return Register();
1156}
1157
1158Register fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1159 if (RetVT.SimpleTy != MVT::v8f32)
1160 return Register();
1161 if ((Subtarget->hasVLX())) {
1162 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
1163 }
1164 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1165 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
1166 }
1167 return Register();
1168}
1169
1170Register fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, Register Op0) {
1171 if (RetVT.SimpleTy != MVT::v16f32)
1172 return Register();
1173 if ((Subtarget->hasAVX512())) {
1174 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
1175 }
1176 return Register();
1177}
1178
1179Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1180 if (RetVT.SimpleTy != MVT::v2f64)
1181 return Register();
1182 if ((Subtarget->hasVLX())) {
1183 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
1184 }
1185 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1186 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
1187 }
1188 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1189 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
1190 }
1191 return Register();
1192}
1193
1194Register fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1195 if (RetVT.SimpleTy != MVT::v4f64)
1196 return Register();
1197 if ((Subtarget->hasVLX())) {
1198 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
1199 }
1200 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1201 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
1202 }
1203 return Register();
1204}
1205
1206Register fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1207 if (RetVT.SimpleTy != MVT::v8f64)
1208 return Register();
1209 if ((Subtarget->hasAVX512())) {
1210 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
1211 }
1212 return Register();
1213}
1214
1215Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
1216 switch (VT.SimpleTy) {
1217 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
1218 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
1219 case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0);
1220 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
1221 case MVT::v16f16: return fastEmit_ISD_FSQRT_MVT_v16f16_r(RetVT, Op0);
1222 case MVT::v32f16: return fastEmit_ISD_FSQRT_MVT_v32f16_r(RetVT, Op0);
1223 case MVT::v8bf16: return fastEmit_ISD_FSQRT_MVT_v8bf16_r(RetVT, Op0);
1224 case MVT::v16bf16: return fastEmit_ISD_FSQRT_MVT_v16bf16_r(RetVT, Op0);
1225 case MVT::v32bf16: return fastEmit_ISD_FSQRT_MVT_v32bf16_r(RetVT, Op0);
1226 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
1227 case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0);
1228 case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0);
1229 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
1230 case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0);
1231 case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0);
1232 default: return Register();
1233 }
1234}
1235
1236// FastEmit functions for ISD::LLRINT.
1237
1238Register fastEmit_ISD_LLRINT_MVT_f16_r(MVT RetVT, Register Op0) {
1239 if (RetVT.SimpleTy != MVT::i64)
1240 return Register();
1241 if ((Subtarget->hasFP16()) && (Subtarget->is64Bit())) {
1242 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1243 }
1244 return Register();
1245}
1246
1247Register fastEmit_ISD_LLRINT_MVT_f32_r(MVT RetVT, Register Op0) {
1248 if (RetVT.SimpleTy != MVT::i64)
1249 return Register();
1250 if ((Subtarget->hasAVX512())) {
1251 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1252 }
1253 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1254 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1255 }
1256 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1257 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1258 }
1259 return Register();
1260}
1261
1262Register fastEmit_ISD_LLRINT_MVT_f64_r(MVT RetVT, Register Op0) {
1263 if (RetVT.SimpleTy != MVT::i64)
1264 return Register();
1265 if ((Subtarget->hasAVX512())) {
1266 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1267 }
1268 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1269 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1270 }
1271 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1272 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1273 }
1274 return Register();
1275}
1276
1277Register fastEmit_ISD_LLRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1278 if (RetVT.SimpleTy != MVT::v8i64)
1279 return Register();
1280 if ((Subtarget->hasFP16())) {
1281 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
1282 }
1283 return Register();
1284}
1285
1286Register fastEmit_ISD_LLRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1287 if (RetVT.SimpleTy != MVT::v4i64)
1288 return Register();
1289 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1290 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1291 }
1292 return Register();
1293}
1294
1295Register fastEmit_ISD_LLRINT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1296 if (RetVT.SimpleTy != MVT::v8i64)
1297 return Register();
1298 if ((Subtarget->hasDQI())) {
1299 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1300 }
1301 return Register();
1302}
1303
1304Register fastEmit_ISD_LLRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1305 if (RetVT.SimpleTy != MVT::v2i64)
1306 return Register();
1307 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1308 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1309 }
1310 return Register();
1311}
1312
1313Register fastEmit_ISD_LLRINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1314 if (RetVT.SimpleTy != MVT::v4i64)
1315 return Register();
1316 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1317 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1318 }
1319 return Register();
1320}
1321
1322Register fastEmit_ISD_LLRINT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1323 if (RetVT.SimpleTy != MVT::v8i64)
1324 return Register();
1325 if ((Subtarget->hasDQI())) {
1326 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1327 }
1328 return Register();
1329}
1330
1331Register fastEmit_ISD_LLRINT_r(MVT VT, MVT RetVT, Register Op0) {
1332 switch (VT.SimpleTy) {
1333 case MVT::f16: return fastEmit_ISD_LLRINT_MVT_f16_r(RetVT, Op0);
1334 case MVT::f32: return fastEmit_ISD_LLRINT_MVT_f32_r(RetVT, Op0);
1335 case MVT::f64: return fastEmit_ISD_LLRINT_MVT_f64_r(RetVT, Op0);
1336 case MVT::v8f16: return fastEmit_ISD_LLRINT_MVT_v8f16_r(RetVT, Op0);
1337 case MVT::v4f32: return fastEmit_ISD_LLRINT_MVT_v4f32_r(RetVT, Op0);
1338 case MVT::v8f32: return fastEmit_ISD_LLRINT_MVT_v8f32_r(RetVT, Op0);
1339 case MVT::v2f64: return fastEmit_ISD_LLRINT_MVT_v2f64_r(RetVT, Op0);
1340 case MVT::v4f64: return fastEmit_ISD_LLRINT_MVT_v4f64_r(RetVT, Op0);
1341 case MVT::v8f64: return fastEmit_ISD_LLRINT_MVT_v8f64_r(RetVT, Op0);
1342 default: return Register();
1343 }
1344}
1345
1346// FastEmit functions for ISD::LRINT.
1347
1348Register fastEmit_ISD_LRINT_MVT_f16_MVT_i32_r(Register Op0) {
1349 if ((Subtarget->hasFP16())) {
1350 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
1351 }
1352 return Register();
1353}
1354
1355Register fastEmit_ISD_LRINT_MVT_f16_MVT_i64_r(Register Op0) {
1356 if ((Subtarget->hasFP16()) && (Subtarget->is64Bit())) {
1357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1358 }
1359 return Register();
1360}
1361
1362Register fastEmit_ISD_LRINT_MVT_f16_r(MVT RetVT, Register Op0) {
1363switch (RetVT.SimpleTy) {
1364 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f16_MVT_i32_r(Op0);
1365 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f16_MVT_i64_r(Op0);
1366 default: return Register();
1367}
1368}
1369
1370Register fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Register Op0) {
1371 if ((Subtarget->hasAVX512())) {
1372 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
1373 }
1374 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1375 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1376 }
1377 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1378 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1379 }
1380 return Register();
1381}
1382
1383Register fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Register Op0) {
1384 if ((Subtarget->hasAVX512())) {
1385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1386 }
1387 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1388 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1389 }
1390 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1391 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1392 }
1393 return Register();
1394}
1395
1396Register fastEmit_ISD_LRINT_MVT_f32_r(MVT RetVT, Register Op0) {
1397switch (RetVT.SimpleTy) {
1398 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Op0);
1399 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Op0);
1400 default: return Register();
1401}
1402}
1403
1404Register fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Register Op0) {
1405 if ((Subtarget->hasAVX512())) {
1406 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
1407 }
1408 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1409 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1410 }
1411 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1412 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1413 }
1414 return Register();
1415}
1416
1417Register fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Register Op0) {
1418 if ((Subtarget->hasAVX512())) {
1419 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1420 }
1421 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1422 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1423 }
1424 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1425 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1426 }
1427 return Register();
1428}
1429
1430Register fastEmit_ISD_LRINT_MVT_f64_r(MVT RetVT, Register Op0) {
1431switch (RetVT.SimpleTy) {
1432 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Op0);
1433 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Op0);
1434 default: return Register();
1435}
1436}
1437
1438Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i16_r(Register Op0) {
1439 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1440 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
1441 }
1442 return Register();
1443}
1444
1445Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i32_r(Register Op0) {
1446 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1447 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1448 }
1449 return Register();
1450}
1451
1452Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i64_r(Register Op0) {
1453 if ((Subtarget->hasFP16())) {
1454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
1455 }
1456 return Register();
1457}
1458
1459Register fastEmit_ISD_LRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1460switch (RetVT.SimpleTy) {
1461 case MVT::v8i16: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i16_r(Op0);
1462 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i32_r(Op0);
1463 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i64_r(Op0);
1464 default: return Register();
1465}
1466}
1467
1468Register fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i16_r(Register Op0) {
1469 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1470 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
1471 }
1472 return Register();
1473}
1474
1475Register fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i32_r(Register Op0) {
1476 if ((Subtarget->hasFP16())) {
1477 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
1478 }
1479 return Register();
1480}
1481
1482Register fastEmit_ISD_LRINT_MVT_v16f16_r(MVT RetVT, Register Op0) {
1483switch (RetVT.SimpleTy) {
1484 case MVT::v16i16: return fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i16_r(Op0);
1485 case MVT::v16i32: return fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i32_r(Op0);
1486 default: return Register();
1487}
1488}
1489
1490Register fastEmit_ISD_LRINT_MVT_v32f16_r(MVT RetVT, Register Op0) {
1491 if (RetVT.SimpleTy != MVT::v32i16)
1492 return Register();
1493 if ((Subtarget->hasFP16())) {
1494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
1495 }
1496 return Register();
1497}
1498
1499Register fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Register Op0) {
1500 if ((Subtarget->hasVLX())) {
1501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
1502 }
1503 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1504 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1505 }
1506 if ((Subtarget->hasAVX())) {
1507 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1508 }
1509 return Register();
1510}
1511
1512Register fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Register Op0) {
1513 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1514 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1515 }
1516 return Register();
1517}
1518
1519Register fastEmit_ISD_LRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1520switch (RetVT.SimpleTy) {
1521 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Op0);
1522 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Op0);
1523 default: return Register();
1524}
1525}
1526
1527Register fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Register Op0) {
1528 if ((Subtarget->hasVLX())) {
1529 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1530 }
1531 if ((Subtarget->hasAVX())) {
1532 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
1533 }
1534 return Register();
1535}
1536
1537Register fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Register Op0) {
1538 if ((Subtarget->hasDQI())) {
1539 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1540 }
1541 return Register();
1542}
1543
1544Register fastEmit_ISD_LRINT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1545switch (RetVT.SimpleTy) {
1546 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Op0);
1547 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Op0);
1548 default: return Register();
1549}
1550}
1551
1552Register fastEmit_ISD_LRINT_MVT_v16f32_r(MVT RetVT, Register Op0) {
1553 if (RetVT.SimpleTy != MVT::v16i32)
1554 return Register();
1555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
1556}
1557
1558Register fastEmit_ISD_LRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1559 if (RetVT.SimpleTy != MVT::v2i64)
1560 return Register();
1561 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1562 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1563 }
1564 return Register();
1565}
1566
1567Register fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Register Op0) {
1568 if ((Subtarget->hasVLX())) {
1569 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
1570 }
1571 if ((Subtarget->hasAVX())) {
1572 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
1573 }
1574 return Register();
1575}
1576
1577Register fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Register Op0) {
1578 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1579 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1580 }
1581 return Register();
1582}
1583
1584Register fastEmit_ISD_LRINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1585switch (RetVT.SimpleTy) {
1586 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Op0);
1587 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Op0);
1588 default: return Register();
1589}
1590}
1591
1592Register fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Register Op0) {
1593 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
1594}
1595
1596Register fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Register Op0) {
1597 if ((Subtarget->hasDQI())) {
1598 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1599 }
1600 return Register();
1601}
1602
1603Register fastEmit_ISD_LRINT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1604switch (RetVT.SimpleTy) {
1605 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Op0);
1606 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Op0);
1607 default: return Register();
1608}
1609}
1610
1611Register fastEmit_ISD_LRINT_r(MVT VT, MVT RetVT, Register Op0) {
1612 switch (VT.SimpleTy) {
1613 case MVT::f16: return fastEmit_ISD_LRINT_MVT_f16_r(RetVT, Op0);
1614 case MVT::f32: return fastEmit_ISD_LRINT_MVT_f32_r(RetVT, Op0);
1615 case MVT::f64: return fastEmit_ISD_LRINT_MVT_f64_r(RetVT, Op0);
1616 case MVT::v8f16: return fastEmit_ISD_LRINT_MVT_v8f16_r(RetVT, Op0);
1617 case MVT::v16f16: return fastEmit_ISD_LRINT_MVT_v16f16_r(RetVT, Op0);
1618 case MVT::v32f16: return fastEmit_ISD_LRINT_MVT_v32f16_r(RetVT, Op0);
1619 case MVT::v4f32: return fastEmit_ISD_LRINT_MVT_v4f32_r(RetVT, Op0);
1620 case MVT::v8f32: return fastEmit_ISD_LRINT_MVT_v8f32_r(RetVT, Op0);
1621 case MVT::v16f32: return fastEmit_ISD_LRINT_MVT_v16f32_r(RetVT, Op0);
1622 case MVT::v2f64: return fastEmit_ISD_LRINT_MVT_v2f64_r(RetVT, Op0);
1623 case MVT::v4f64: return fastEmit_ISD_LRINT_MVT_v4f64_r(RetVT, Op0);
1624 case MVT::v8f64: return fastEmit_ISD_LRINT_MVT_v8f64_r(RetVT, Op0);
1625 default: return Register();
1626 }
1627}
1628
1629// FastEmit functions for ISD::SCALAR_TO_VECTOR.
1630
1631Register fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
1632 if (RetVT.SimpleTy != MVT::v4i32)
1633 return Register();
1634 if ((Subtarget->hasAVX512())) {
1635 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIZrr, RC: &X86::VR128XRegClass, Op0);
1636 }
1637 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1638 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1639 }
1640 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1641 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1642 }
1643 return Register();
1644}
1645
1646Register fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
1647 if (RetVT.SimpleTy != MVT::v2i64)
1648 return Register();
1649 if ((Subtarget->hasAVX512())) {
1650 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIZrr, RC: &X86::VR128XRegClass, Op0);
1651 }
1652 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1653 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1654 }
1655 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1656 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1657 }
1658 return Register();
1659}
1660
1661Register fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
1662 switch (VT.SimpleTy) {
1663 case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0);
1664 case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0);
1665 default: return Register();
1666 }
1667}
1668
1669// FastEmit functions for ISD::SIGN_EXTEND.
1670
1671Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Register Op0) {
1672 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr8, RC: &X86::GR32RegClass, Op0);
1673}
1674
1675Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Register Op0) {
1676 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr8, RC: &X86::GR64RegClass, Op0);
1677}
1678
1679Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
1680switch (RetVT.SimpleTy) {
1681 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0);
1682 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0);
1683 default: return Register();
1684}
1685}
1686
1687Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Register Op0) {
1688 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr16, RC: &X86::GR32RegClass, Op0);
1689}
1690
1691Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Register Op0) {
1692 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr16, RC: &X86::GR64RegClass, Op0);
1693}
1694
1695Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, Register Op0) {
1696switch (RetVT.SimpleTy) {
1697 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0);
1698 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0);
1699 default: return Register();
1700}
1701}
1702
1703Register fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, Register Op0) {
1704 if (RetVT.SimpleTy != MVT::i64)
1705 return Register();
1706 if ((Subtarget->is64Bit())) {
1707 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr32, RC: &X86::GR64RegClass, Op0);
1708 }
1709 return Register();
1710}
1711
1712Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, Register Op0) {
1713 if (RetVT.SimpleTy != MVT::v2i64)
1714 return Register();
1715 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1716 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ128rk, RC: &X86::VR128XRegClass, Op0);
1717 }
1718 return Register();
1719}
1720
1721Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Register Op0) {
1722 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1723 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ128rk, RC: &X86::VR128XRegClass, Op0);
1724 }
1725 return Register();
1726}
1727
1728Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Register Op0) {
1729 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1730 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ256rk, RC: &X86::VR256XRegClass, Op0);
1731 }
1732 return Register();
1733}
1734
1735Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, Register Op0) {
1736switch (RetVT.SimpleTy) {
1737 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0);
1738 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0);
1739 default: return Register();
1740}
1741}
1742
1743Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Register Op0) {
1744 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1745 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ128rk, RC: &X86::VR128XRegClass, Op0);
1746 }
1747 return Register();
1748}
1749
1750Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Register Op0) {
1751 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1752 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ256rk, RC: &X86::VR256XRegClass, Op0);
1753 }
1754 return Register();
1755}
1756
1757Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Register Op0) {
1758 if ((Subtarget->hasDQI())) {
1759 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZrk, RC: &X86::VR512RegClass, Op0);
1760 }
1761 return Register();
1762}
1763
1764Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, Register Op0) {
1765switch (RetVT.SimpleTy) {
1766 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0);
1767 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0);
1768 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0);
1769 default: return Register();
1770}
1771}
1772
1773Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Register Op0) {
1774 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1775 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ128rk, RC: &X86::VR128XRegClass, Op0);
1776 }
1777 return Register();
1778}
1779
1780Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Register Op0) {
1781 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1782 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ256rk, RC: &X86::VR256XRegClass, Op0);
1783 }
1784 return Register();
1785}
1786
1787Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Register Op0) {
1788 if ((Subtarget->hasDQI())) {
1789 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZrk, RC: &X86::VR512RegClass, Op0);
1790 }
1791 return Register();
1792}
1793
1794Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, Register Op0) {
1795switch (RetVT.SimpleTy) {
1796 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0);
1797 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0);
1798 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0);
1799 default: return Register();
1800}
1801}
1802
1803Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Register Op0) {
1804 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1805 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ256rk, RC: &X86::VR256XRegClass, Op0);
1806 }
1807 return Register();
1808}
1809
1810Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Register Op0) {
1811 if ((Subtarget->hasBWI())) {
1812 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZrk, RC: &X86::VR512RegClass, Op0);
1813 }
1814 return Register();
1815}
1816
1817Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, Register Op0) {
1818switch (RetVT.SimpleTy) {
1819 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0);
1820 case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0);
1821 default: return Register();
1822}
1823}
1824
1825Register fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, Register Op0) {
1826 if (RetVT.SimpleTy != MVT::v64i8)
1827 return Register();
1828 if ((Subtarget->hasBWI())) {
1829 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZrk, RC: &X86::VR512RegClass, Op0);
1830 }
1831 return Register();
1832}
1833
1834Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Register Op0) {
1835 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1836 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWYrr, RC: &X86::VR256RegClass, Op0);
1837 }
1838 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1839 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
1840 }
1841 return Register();
1842}
1843
1844Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Register Op0) {
1845 if ((Subtarget->hasAVX512())) {
1846 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZrr, RC: &X86::VR512RegClass, Op0);
1847 }
1848 return Register();
1849}
1850
1851Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(MVT RetVT, Register Op0) {
1852switch (RetVT.SimpleTy) {
1853 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
1854 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
1855 default: return Register();
1856}
1857}
1858
1859Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(MVT RetVT, Register Op0) {
1860 if (RetVT.SimpleTy != MVT::v32i16)
1861 return Register();
1862 if ((Subtarget->hasBWI())) {
1863 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZrr, RC: &X86::VR512RegClass, Op0);
1864 }
1865 return Register();
1866}
1867
1868Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Register Op0) {
1869 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1870 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDYrr, RC: &X86::VR256RegClass, Op0);
1871 }
1872 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1873 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
1874 }
1875 return Register();
1876}
1877
1878Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Register Op0) {
1879 if ((Subtarget->hasAVX512())) {
1880 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZrr, RC: &X86::VR512RegClass, Op0);
1881 }
1882 return Register();
1883}
1884
1885Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(MVT RetVT, Register Op0) {
1886switch (RetVT.SimpleTy) {
1887 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
1888 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
1889 default: return Register();
1890}
1891}
1892
1893Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(MVT RetVT, Register Op0) {
1894 if (RetVT.SimpleTy != MVT::v16i32)
1895 return Register();
1896 if ((Subtarget->hasAVX512())) {
1897 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZrr, RC: &X86::VR512RegClass, Op0);
1898 }
1899 return Register();
1900}
1901
1902Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(MVT RetVT, Register Op0) {
1903 if (RetVT.SimpleTy != MVT::v4i64)
1904 return Register();
1905 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1906 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQYrr, RC: &X86::VR256RegClass, Op0);
1907 }
1908 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1909 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
1910 }
1911 return Register();
1912}
1913
1914Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(MVT RetVT, Register Op0) {
1915 if (RetVT.SimpleTy != MVT::v8i64)
1916 return Register();
1917 if ((Subtarget->hasAVX512())) {
1918 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZrr, RC: &X86::VR512RegClass, Op0);
1919 }
1920 return Register();
1921}
1922
1923Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
1924 switch (VT.SimpleTy) {
1925 case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0);
1926 case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0);
1927 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0);
1928 case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0);
1929 case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0);
1930 case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0);
1931 case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0);
1932 case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0);
1933 case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0);
1934 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(RetVT, Op0);
1935 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(RetVT, Op0);
1936 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(RetVT, Op0);
1937 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(RetVT, Op0);
1938 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(RetVT, Op0);
1939 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(RetVT, Op0);
1940 default: return Register();
1941 }
1942}
1943
1944// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
1945
1946Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1947 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1948 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1949 }
1950 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1951 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1952 }
1953 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1954 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
1955 }
1956 return Register();
1957}
1958
1959Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1960 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1961 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
1962 }
1963 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1964 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
1965 }
1966 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1967 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
1968 }
1969 return Register();
1970}
1971
1972Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Register Op0) {
1973 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1974 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDYrr, RC: &X86::VR256RegClass, Op0);
1975 }
1976 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1977 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
1978 }
1979 return Register();
1980}
1981
1982Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1983 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1984 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
1985 }
1986 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1987 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
1988 }
1989 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1990 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
1991 }
1992 return Register();
1993}
1994
1995Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Register Op0) {
1996 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1997 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQYrr, RC: &X86::VR256RegClass, Op0);
1998 }
1999 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2000 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
2001 }
2002 return Register();
2003}
2004
2005Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Register Op0) {
2006 if ((Subtarget->hasAVX512())) {
2007 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZrr, RC: &X86::VR512RegClass, Op0);
2008 }
2009 return Register();
2010}
2011
2012Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, Register Op0) {
2013switch (RetVT.SimpleTy) {
2014 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
2015 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
2016 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
2017 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
2018 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
2019 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
2020 default: return Register();
2021}
2022}
2023
2024Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Register Op0) {
2025 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2026 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
2027 }
2028 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2029 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
2030 }
2031 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2032 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
2033 }
2034 return Register();
2035}
2036
2037Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Register Op0) {
2038 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2039 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
2040 }
2041 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2042 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
2043 }
2044 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2045 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
2046 }
2047 return Register();
2048}
2049
2050Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Register Op0) {
2051 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2052 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQYrr, RC: &X86::VR256RegClass, Op0);
2053 }
2054 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2055 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
2056 }
2057 return Register();
2058}
2059
2060Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, Register Op0) {
2061switch (RetVT.SimpleTy) {
2062 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
2063 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
2064 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
2065 default: return Register();
2066}
2067}
2068
2069Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, Register Op0) {
2070 if (RetVT.SimpleTy != MVT::v2i64)
2071 return Register();
2072 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2073 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
2074 }
2075 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2076 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
2077 }
2078 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2079 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
2080 }
2081 return Register();
2082}
2083
2084Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, Register Op0) {
2085 switch (VT.SimpleTy) {
2086 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
2087 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
2088 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
2089 default: return Register();
2090 }
2091}
2092
2093// FastEmit functions for ISD::SINT_TO_FP.
2094
2095Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
2096 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2097 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2098 }
2099 return Register();
2100}
2101
2102Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
2103 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2104 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2105 }
2106 return Register();
2107}
2108
2109Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
2110switch (RetVT.SimpleTy) {
2111 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2112 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2113 default: return Register();
2114}
2115}
2116
2117Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
2118 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2119 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2120 }
2121 return Register();
2122}
2123
2124Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
2125 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2126 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2127 }
2128 return Register();
2129}
2130
2131Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
2132switch (RetVT.SimpleTy) {
2133 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2134 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2135 default: return Register();
2136}
2137}
2138
2139Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2140 if (RetVT.SimpleTy != MVT::v8f16)
2141 return Register();
2142 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2144 }
2145 return Register();
2146}
2147
2148Register fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
2149 if (RetVT.SimpleTy != MVT::v16f16)
2150 return Register();
2151 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2152 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2153 }
2154 return Register();
2155}
2156
2157Register fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
2158 if (RetVT.SimpleTy != MVT::v32f16)
2159 return Register();
2160 if ((Subtarget->hasFP16())) {
2161 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2162 }
2163 return Register();
2164}
2165
2166Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
2167 if ((Subtarget->hasVLX())) {
2168 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2169 }
2170 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2171 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2172 }
2173 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2174 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2175 }
2176 return Register();
2177}
2178
2179Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
2180 if ((Subtarget->hasVLX())) {
2181 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2182 }
2183 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2184 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2185 }
2186 return Register();
2187}
2188
2189Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2190switch (RetVT.SimpleTy) {
2191 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2192 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2193 default: return Register();
2194}
2195}
2196
2197Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
2198 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2199 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2200 }
2201 return Register();
2202}
2203
2204Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
2205 if ((Subtarget->hasVLX())) {
2206 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2207 }
2208 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2210 }
2211 return Register();
2212}
2213
2214Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
2215 if ((Subtarget->hasAVX512())) {
2216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2217 }
2218 return Register();
2219}
2220
2221Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
2222switch (RetVT.SimpleTy) {
2223 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2224 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2225 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2226 default: return Register();
2227}
2228}
2229
2230Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
2231 if ((Subtarget->hasFP16())) {
2232 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2233 }
2234 return Register();
2235}
2236
2237Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
2238 if ((Subtarget->hasAVX512())) {
2239 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2240 }
2241 return Register();
2242}
2243
2244Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
2245switch (RetVT.SimpleTy) {
2246 case MVT::v16f16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2247 case MVT::v16f32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2248 default: return Register();
2249}
2250}
2251
2252Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2253 if (RetVT.SimpleTy != MVT::v2f64)
2254 return Register();
2255 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2256 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2257 }
2258 return Register();
2259}
2260
2261Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
2262 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2263 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2264 }
2265 return Register();
2266}
2267
2268Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
2269 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2270 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2271 }
2272 return Register();
2273}
2274
2275Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
2276switch (RetVT.SimpleTy) {
2277 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2278 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2279 default: return Register();
2280}
2281}
2282
2283Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
2284 if ((Subtarget->hasFP16())) {
2285 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2286 }
2287 return Register();
2288}
2289
2290Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
2291 if ((Subtarget->hasDQI())) {
2292 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2293 }
2294 return Register();
2295}
2296
2297Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
2298 if ((Subtarget->hasDQI())) {
2299 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2300 }
2301 return Register();
2302}
2303
2304Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
2305switch (RetVT.SimpleTy) {
2306 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
2307 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
2308 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
2309 default: return Register();
2310}
2311}
2312
2313Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
2314 switch (VT.SimpleTy) {
2315 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
2316 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
2317 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2318 case MVT::v16i16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
2319 case MVT::v32i16: return fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
2320 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2321 case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
2322 case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
2323 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
2324 case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
2325 case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
2326 default: return Register();
2327 }
2328}
2329
2330// FastEmit functions for ISD::STRICT_FP_EXTEND.
2331
2332Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2333 if (RetVT.SimpleTy != MVT::f64)
2334 return Register();
2335 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2336 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
2337 }
2338 return Register();
2339}
2340
2341Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Register Op0) {
2342 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2343 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
2344 }
2345 return Register();
2346}
2347
2348Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Register Op0) {
2349 if ((Subtarget->hasFP16())) {
2350 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
2351 }
2352 return Register();
2353}
2354
2355Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2356switch (RetVT.SimpleTy) {
2357 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
2358 case MVT::v8f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
2359 default: return Register();
2360}
2361}
2362
2363Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(MVT RetVT, Register Op0) {
2364 if (RetVT.SimpleTy != MVT::v16f32)
2365 return Register();
2366 if ((Subtarget->hasFP16())) {
2367 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
2368 }
2369 return Register();
2370}
2371
2372Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2373 if (RetVT.SimpleTy != MVT::v4f64)
2374 return Register();
2375 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2376 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2377 }
2378 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2379 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
2380 }
2381 return Register();
2382}
2383
2384Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(MVT RetVT, Register Op0) {
2385 if (RetVT.SimpleTy != MVT::v8f64)
2386 return Register();
2387 if ((Subtarget->hasAVX512())) {
2388 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
2389 }
2390 return Register();
2391}
2392
2393Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2394 switch (VT.SimpleTy) {
2395 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2396 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
2397 case MVT::v16f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
2398 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
2399 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
2400 default: return Register();
2401 }
2402}
2403
2404// FastEmit functions for ISD::STRICT_FP_ROUND.
2405
2406Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2407 if (RetVT.SimpleTy != MVT::f32)
2408 return Register();
2409 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2410 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
2411 }
2412 return Register();
2413}
2414
2415Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2416 switch (VT.SimpleTy) {
2417 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
2418 default: return Register();
2419 }
2420}
2421
2422// FastEmit functions for ISD::STRICT_FP_TO_SINT.
2423
2424Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
2425 if ((Subtarget->hasFP16())) {
2426 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
2427 }
2428 return Register();
2429}
2430
2431Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
2432 if ((Subtarget->hasFP16())) {
2433 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2434 }
2435 return Register();
2436}
2437
2438Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
2439switch (RetVT.SimpleTy) {
2440 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
2441 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
2442 default: return Register();
2443}
2444}
2445
2446Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
2447 if ((Subtarget->hasAVX512())) {
2448 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
2449 }
2450 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2451 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2452 }
2453 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2455 }
2456 return Register();
2457}
2458
2459Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
2460 if ((Subtarget->hasAVX512())) {
2461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2462 }
2463 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2464 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2465 }
2466 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2467 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2468 }
2469 return Register();
2470}
2471
2472Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
2473switch (RetVT.SimpleTy) {
2474 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
2475 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
2476 default: return Register();
2477}
2478}
2479
2480Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
2481 if ((Subtarget->hasAVX512())) {
2482 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
2483 }
2484 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2485 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2486 }
2487 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2488 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2489 }
2490 return Register();
2491}
2492
2493Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
2494 if ((Subtarget->hasAVX512())) {
2495 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2496 }
2497 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2498 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2499 }
2500 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2502 }
2503 return Register();
2504}
2505
2506Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
2507switch (RetVT.SimpleTy) {
2508 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
2509 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
2510 default: return Register();
2511}
2512}
2513
2514Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
2515 if (RetVT.SimpleTy != MVT::v4i32)
2516 return Register();
2517 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2518 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
2519 }
2520 return Register();
2521}
2522
2523Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2524 switch (VT.SimpleTy) {
2525 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
2526 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
2527 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
2528 case MVT::v4f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
2529 default: return Register();
2530 }
2531}
2532
2533// FastEmit functions for ISD::STRICT_FP_TO_UINT.
2534
2535Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
2536 if ((Subtarget->hasFP16())) {
2537 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
2538 }
2539 return Register();
2540}
2541
2542Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
2543 if ((Subtarget->hasFP16())) {
2544 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2545 }
2546 return Register();
2547}
2548
2549Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
2550switch (RetVT.SimpleTy) {
2551 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
2552 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
2553 default: return Register();
2554}
2555}
2556
2557Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
2558 if ((Subtarget->hasAVX512())) {
2559 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
2560 }
2561 return Register();
2562}
2563
2564Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
2565 if ((Subtarget->hasAVX512())) {
2566 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2567 }
2568 return Register();
2569}
2570
2571Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
2572switch (RetVT.SimpleTy) {
2573 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
2574 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
2575 default: return Register();
2576}
2577}
2578
2579Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
2580 if ((Subtarget->hasAVX512())) {
2581 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
2582 }
2583 return Register();
2584}
2585
2586Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
2587 if ((Subtarget->hasAVX512())) {
2588 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2589 }
2590 return Register();
2591}
2592
2593Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
2594switch (RetVT.SimpleTy) {
2595 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
2596 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
2597 default: return Register();
2598}
2599}
2600
2601Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2602 switch (VT.SimpleTy) {
2603 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
2604 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
2605 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
2606 default: return Register();
2607 }
2608}
2609
2610// FastEmit functions for ISD::STRICT_FSQRT.
2611
2612Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
2613 if (RetVT.SimpleTy != MVT::f32)
2614 return Register();
2615 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2616 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
2617 }
2618 if ((!Subtarget->hasSSE1())) {
2619 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
2620 }
2621 return Register();
2622}
2623
2624Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
2625 if (RetVT.SimpleTy != MVT::f64)
2626 return Register();
2627 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2628 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
2629 }
2630 if ((!Subtarget->hasSSE2())) {
2631 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
2632 }
2633 return Register();
2634}
2635
2636Register fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(MVT RetVT, Register Op0) {
2637 if (RetVT.SimpleTy != MVT::f80)
2638 return Register();
2639 if ((Subtarget->hasX87())) {
2640 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
2641 }
2642 return Register();
2643}
2644
2645Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2646 if (RetVT.SimpleTy != MVT::v8f16)
2647 return Register();
2648 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2649 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
2650 }
2651 return Register();
2652}
2653
2654Register fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(MVT RetVT, Register Op0) {
2655 if (RetVT.SimpleTy != MVT::v16f16)
2656 return Register();
2657 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2658 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
2659 }
2660 return Register();
2661}
2662
2663Register fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(MVT RetVT, Register Op0) {
2664 if (RetVT.SimpleTy != MVT::v32f16)
2665 return Register();
2666 if ((Subtarget->hasFP16())) {
2667 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
2668 }
2669 return Register();
2670}
2671
2672Register fastEmit_ISD_STRICT_FSQRT_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2673 if (RetVT.SimpleTy != MVT::v8bf16)
2674 return Register();
2675 if ((Subtarget->hasAVX10_2())) {
2676 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
2677 }
2678 return Register();
2679}
2680
2681Register fastEmit_ISD_STRICT_FSQRT_MVT_v16bf16_r(MVT RetVT, Register Op0) {
2682 if (RetVT.SimpleTy != MVT::v16bf16)
2683 return Register();
2684 if ((Subtarget->hasAVX10_2())) {
2685 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
2686 }
2687 return Register();
2688}
2689
2690Register fastEmit_ISD_STRICT_FSQRT_MVT_v32bf16_r(MVT RetVT, Register Op0) {
2691 if (RetVT.SimpleTy != MVT::v32bf16)
2692 return Register();
2693 if ((Subtarget->hasAVX10_2())) {
2694 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
2695 }
2696 return Register();
2697}
2698
2699Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2700 if (RetVT.SimpleTy != MVT::v4f32)
2701 return Register();
2702 if ((Subtarget->hasVLX())) {
2703 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
2704 }
2705 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2706 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
2707 }
2708 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2709 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
2710 }
2711 return Register();
2712}
2713
2714Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
2715 if (RetVT.SimpleTy != MVT::v8f32)
2716 return Register();
2717 if ((Subtarget->hasVLX())) {
2718 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
2719 }
2720 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2721 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
2722 }
2723 return Register();
2724}
2725
2726Register fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(MVT RetVT, Register Op0) {
2727 if (RetVT.SimpleTy != MVT::v16f32)
2728 return Register();
2729 if ((Subtarget->hasAVX512())) {
2730 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
2731 }
2732 return Register();
2733}
2734
2735Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2736 if (RetVT.SimpleTy != MVT::v2f64)
2737 return Register();
2738 if ((Subtarget->hasVLX())) {
2739 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
2740 }
2741 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2742 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
2743 }
2744 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2745 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
2746 }
2747 return Register();
2748}
2749
2750Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(MVT RetVT, Register Op0) {
2751 if (RetVT.SimpleTy != MVT::v4f64)
2752 return Register();
2753 if ((Subtarget->hasVLX())) {
2754 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
2755 }
2756 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2757 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
2758 }
2759 return Register();
2760}
2761
2762Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(MVT RetVT, Register Op0) {
2763 if (RetVT.SimpleTy != MVT::v8f64)
2764 return Register();
2765 if ((Subtarget->hasAVX512())) {
2766 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
2767 }
2768 return Register();
2769}
2770
2771Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
2772 switch (VT.SimpleTy) {
2773 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
2774 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
2775 case MVT::f80: return fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(RetVT, Op0);
2776 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
2777 case MVT::v16f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(RetVT, Op0);
2778 case MVT::v32f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(RetVT, Op0);
2779 case MVT::v8bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8bf16_r(RetVT, Op0);
2780 case MVT::v16bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16bf16_r(RetVT, Op0);
2781 case MVT::v32bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32bf16_r(RetVT, Op0);
2782 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
2783 case MVT::v8f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(RetVT, Op0);
2784 case MVT::v16f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(RetVT, Op0);
2785 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
2786 case MVT::v4f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(RetVT, Op0);
2787 case MVT::v8f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(RetVT, Op0);
2788 default: return Register();
2789 }
2790}
2791
2792// FastEmit functions for ISD::STRICT_SINT_TO_FP.
2793
2794Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
2795 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2796 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2797 }
2798 return Register();
2799}
2800
2801Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
2802 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2803 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2804 }
2805 return Register();
2806}
2807
2808Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
2809switch (RetVT.SimpleTy) {
2810 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2811 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2812 default: return Register();
2813}
2814}
2815
2816Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
2817 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2818 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2819 }
2820 return Register();
2821}
2822
2823Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
2824 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2825 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2826 }
2827 return Register();
2828}
2829
2830Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
2831switch (RetVT.SimpleTy) {
2832 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2833 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2834 default: return Register();
2835}
2836}
2837
2838Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2839 if (RetVT.SimpleTy != MVT::v8f16)
2840 return Register();
2841 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2842 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2843 }
2844 return Register();
2845}
2846
2847Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
2848 if (RetVT.SimpleTy != MVT::v16f16)
2849 return Register();
2850 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2851 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2852 }
2853 return Register();
2854}
2855
2856Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
2857 if (RetVT.SimpleTy != MVT::v32f16)
2858 return Register();
2859 if ((Subtarget->hasFP16())) {
2860 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2861 }
2862 return Register();
2863}
2864
2865Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
2866 if ((Subtarget->hasVLX())) {
2867 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2868 }
2869 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2870 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2871 }
2872 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2873 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2874 }
2875 return Register();
2876}
2877
2878Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
2879 if ((Subtarget->hasVLX())) {
2880 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2881 }
2882 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2883 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2884 }
2885 return Register();
2886}
2887
2888Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2889switch (RetVT.SimpleTy) {
2890 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2891 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2892 default: return Register();
2893}
2894}
2895
2896Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
2897 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2898 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2899 }
2900 return Register();
2901}
2902
2903Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
2904 if ((Subtarget->hasVLX())) {
2905 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2906 }
2907 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2908 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2909 }
2910 return Register();
2911}
2912
2913Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
2914 if ((Subtarget->hasAVX512())) {
2915 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2916 }
2917 return Register();
2918}
2919
2920Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
2921switch (RetVT.SimpleTy) {
2922 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2923 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2924 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2925 default: return Register();
2926}
2927}
2928
2929Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
2930 if ((Subtarget->hasFP16())) {
2931 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2932 }
2933 return Register();
2934}
2935
2936Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
2937 if ((Subtarget->hasAVX512())) {
2938 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2939 }
2940 return Register();
2941}
2942
2943Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
2944switch (RetVT.SimpleTy) {
2945 case MVT::v16f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2946 case MVT::v16f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2947 default: return Register();
2948}
2949}
2950
2951Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2952 if (RetVT.SimpleTy != MVT::v2f64)
2953 return Register();
2954 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2955 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2956 }
2957 return Register();
2958}
2959
2960Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
2961 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2963 }
2964 return Register();
2965}
2966
2967Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
2968 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2969 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2970 }
2971 return Register();
2972}
2973
2974Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
2975switch (RetVT.SimpleTy) {
2976 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2977 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2978 default: return Register();
2979}
2980}
2981
2982Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
2983 if ((Subtarget->hasFP16())) {
2984 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2985 }
2986 return Register();
2987}
2988
2989Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
2990 if ((Subtarget->hasDQI())) {
2991 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2992 }
2993 return Register();
2994}
2995
2996Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
2997 if ((Subtarget->hasDQI())) {
2998 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2999 }
3000 return Register();
3001}
3002
3003Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3004switch (RetVT.SimpleTy) {
3005 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3006 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3007 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3008 default: return Register();
3009}
3010}
3011
3012Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3013 switch (VT.SimpleTy) {
3014 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
3015 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
3016 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3017 case MVT::v16i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3018 case MVT::v32i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3019 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3020 case MVT::v8i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3021 case MVT::v16i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3022 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3023 case MVT::v4i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3024 case MVT::v8i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3025 default: return Register();
3026 }
3027}
3028
3029// FastEmit functions for ISD::STRICT_UINT_TO_FP.
3030
3031Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3032 if (RetVT.SimpleTy != MVT::v8f16)
3033 return Register();
3034 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3035 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3036 }
3037 return Register();
3038}
3039
3040Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
3041 if (RetVT.SimpleTy != MVT::v16f16)
3042 return Register();
3043 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3044 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3045 }
3046 return Register();
3047}
3048
3049Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
3050 if (RetVT.SimpleTy != MVT::v32f16)
3051 return Register();
3052 if ((Subtarget->hasFP16())) {
3053 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3054 }
3055 return Register();
3056}
3057
3058Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
3059 if ((Subtarget->hasVLX())) {
3060 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3061 }
3062 return Register();
3063}
3064
3065Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
3066 if ((Subtarget->hasVLX())) {
3067 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3068 }
3069 return Register();
3070}
3071
3072Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3073switch (RetVT.SimpleTy) {
3074 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3075 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3076 default: return Register();
3077}
3078}
3079
3080Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
3081 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3082 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3083 }
3084 return Register();
3085}
3086
3087Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
3088 if ((Subtarget->hasVLX())) {
3089 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3090 }
3091 return Register();
3092}
3093
3094Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
3095 if ((Subtarget->hasAVX512())) {
3096 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3097 }
3098 return Register();
3099}
3100
3101Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
3102switch (RetVT.SimpleTy) {
3103 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3104 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3105 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3106 default: return Register();
3107}
3108}
3109
3110Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
3111 if ((Subtarget->hasFP16())) {
3112 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3113 }
3114 return Register();
3115}
3116
3117Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
3118 if ((Subtarget->hasAVX512())) {
3119 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3120 }
3121 return Register();
3122}
3123
3124Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
3125switch (RetVT.SimpleTy) {
3126 case MVT::v16f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3127 case MVT::v16f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3128 default: return Register();
3129}
3130}
3131
3132Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
3133 if (RetVT.SimpleTy != MVT::v2f64)
3134 return Register();
3135 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3136 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3137 }
3138 return Register();
3139}
3140
3141Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
3142 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3144 }
3145 return Register();
3146}
3147
3148Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3149 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3150 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3151 }
3152 return Register();
3153}
3154
3155Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3156switch (RetVT.SimpleTy) {
3157 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3158 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3159 default: return Register();
3160}
3161}
3162
3163Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3164 if ((Subtarget->hasFP16())) {
3165 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3166 }
3167 return Register();
3168}
3169
3170Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3171 if ((Subtarget->hasDQI())) {
3172 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3173 }
3174 return Register();
3175}
3176
3177Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3178 if ((Subtarget->hasDQI())) {
3179 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3180 }
3181 return Register();
3182}
3183
3184Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3185switch (RetVT.SimpleTy) {
3186 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3187 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3188 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3189 default: return Register();
3190}
3191}
3192
3193Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3194 switch (VT.SimpleTy) {
3195 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3196 case MVT::v16i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3197 case MVT::v32i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3198 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3199 case MVT::v8i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3200 case MVT::v16i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3201 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3202 case MVT::v4i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3203 case MVT::v8i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3204 default: return Register();
3205 }
3206}
3207
3208// FastEmit functions for ISD::TRUNCATE.
3209
3210Register fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, Register Op0) {
3211 if (RetVT.SimpleTy != MVT::i8)
3212 return Register();
3213 if ((Subtarget->is64Bit())) {
3214 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: X86::sub_8bit);
3215 }
3216 return Register();
3217}
3218
3219Register fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Register Op0) {
3220 if ((Subtarget->is64Bit())) {
3221 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3222 }
3223 return Register();
3224}
3225
3226Register fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Register Op0) {
3227 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3228}
3229
3230Register fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, Register Op0) {
3231switch (RetVT.SimpleTy) {
3232 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0);
3233 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0);
3234 default: return Register();
3235}
3236}
3237
3238Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Register Op0) {
3239 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3240}
3241
3242Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Register Op0) {
3243 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3244}
3245
3246Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Register Op0) {
3247 return fastEmitInst_extractsubreg(RetVT: MVT::i32, Op0, Idx: X86::sub_32bit);
3248}
3249
3250Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
3251switch (RetVT.SimpleTy) {
3252 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0);
3253 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0);
3254 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0);
3255 default: return Register();
3256}
3257}
3258
3259Register fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, Register Op0) {
3260 if (RetVT.SimpleTy != MVT::v16i8)
3261 return Register();
3262 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3263 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ256rr, RC: &X86::VR128XRegClass, Op0);
3264 }
3265 return Register();
3266}
3267
3268Register fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, Register Op0) {
3269 if (RetVT.SimpleTy != MVT::v32i8)
3270 return Register();
3271 if ((Subtarget->hasBWI())) {
3272 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZrr, RC: &X86::VR256XRegClass, Op0);
3273 }
3274 return Register();
3275}
3276
3277Register fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, Register Op0) {
3278 if (RetVT.SimpleTy != MVT::v8i16)
3279 return Register();
3280 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3281 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ256rr, RC: &X86::VR128XRegClass, Op0);
3282 }
3283 return Register();
3284}
3285
3286Register fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Register Op0) {
3287 if ((Subtarget->hasAVX512())) {
3288 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZrr, RC: &X86::VR128XRegClass, Op0);
3289 }
3290 return Register();
3291}
3292
3293Register fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Register Op0) {
3294 if ((Subtarget->hasAVX512())) {
3295 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZrr, RC: &X86::VR256XRegClass, Op0);
3296 }
3297 return Register();
3298}
3299
3300Register fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, Register Op0) {
3301switch (RetVT.SimpleTy) {
3302 case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0);
3303 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0);
3304 default: return Register();
3305}
3306}
3307
3308Register fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, Register Op0) {
3309 if (RetVT.SimpleTy != MVT::v4i32)
3310 return Register();
3311 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3312 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ256rr, RC: &X86::VR128XRegClass, Op0);
3313 }
3314 return Register();
3315}
3316
3317Register fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Register Op0) {
3318 if ((Subtarget->hasAVX512())) {
3319 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZrr, RC: &X86::VR128XRegClass, Op0);
3320 }
3321 return Register();
3322}
3323
3324Register fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Register Op0) {
3325 if ((Subtarget->hasAVX512())) {
3326 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZrr, RC: &X86::VR256XRegClass, Op0);
3327 }
3328 return Register();
3329}
3330
3331Register fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, Register Op0) {
3332switch (RetVT.SimpleTy) {
3333 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0);
3334 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0);
3335 default: return Register();
3336}
3337}
3338
3339Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3340 switch (VT.SimpleTy) {
3341 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0);
3342 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0);
3343 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
3344 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0);
3345 case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0);
3346 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0);
3347 case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0);
3348 case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0);
3349 case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0);
3350 default: return Register();
3351 }
3352}
3353
3354// FastEmit functions for ISD::UINT_TO_FP.
3355
3356Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3357 if (RetVT.SimpleTy != MVT::v8f16)
3358 return Register();
3359 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3360 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3361 }
3362 return Register();
3363}
3364
3365Register fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
3366 if (RetVT.SimpleTy != MVT::v16f16)
3367 return Register();
3368 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3369 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3370 }
3371 return Register();
3372}
3373
3374Register fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
3375 if (RetVT.SimpleTy != MVT::v32f16)
3376 return Register();
3377 if ((Subtarget->hasFP16())) {
3378 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3379 }
3380 return Register();
3381}
3382
3383Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
3384 if ((Subtarget->hasVLX())) {
3385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3386 }
3387 return Register();
3388}
3389
3390Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
3391 if ((Subtarget->hasVLX())) {
3392 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3393 }
3394 return Register();
3395}
3396
3397Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3398switch (RetVT.SimpleTy) {
3399 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3400 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3401 default: return Register();
3402}
3403}
3404
3405Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
3406 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3407 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3408 }
3409 return Register();
3410}
3411
3412Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
3413 if ((Subtarget->hasVLX())) {
3414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3415 }
3416 return Register();
3417}
3418
3419Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
3420 if ((Subtarget->hasAVX512())) {
3421 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3422 }
3423 return Register();
3424}
3425
3426Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
3427switch (RetVT.SimpleTy) {
3428 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3429 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3430 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3431 default: return Register();
3432}
3433}
3434
3435Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
3436 if ((Subtarget->hasFP16())) {
3437 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3438 }
3439 return Register();
3440}
3441
3442Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
3443 if ((Subtarget->hasAVX512())) {
3444 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3445 }
3446 return Register();
3447}
3448
3449Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
3450switch (RetVT.SimpleTy) {
3451 case MVT::v16f16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3452 case MVT::v16f32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3453 default: return Register();
3454}
3455}
3456
3457Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
3458 if (RetVT.SimpleTy != MVT::v2f64)
3459 return Register();
3460 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3462 }
3463 return Register();
3464}
3465
3466Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
3467 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3469 }
3470 return Register();
3471}
3472
3473Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3474 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3475 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3476 }
3477 return Register();
3478}
3479
3480Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3481switch (RetVT.SimpleTy) {
3482 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3483 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3484 default: return Register();
3485}
3486}
3487
3488Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3489 if ((Subtarget->hasFP16())) {
3490 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3491 }
3492 return Register();
3493}
3494
3495Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3496 if ((Subtarget->hasDQI())) {
3497 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3498 }
3499 return Register();
3500}
3501
3502Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3503 if ((Subtarget->hasDQI())) {
3504 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3505 }
3506 return Register();
3507}
3508
3509Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3510switch (RetVT.SimpleTy) {
3511 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3512 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3513 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3514 default: return Register();
3515}
3516}
3517
3518Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3519 switch (VT.SimpleTy) {
3520 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3521 case MVT::v16i16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3522 case MVT::v32i16: return fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3523 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3524 case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3525 case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3526 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3527 case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3528 case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3529 default: return Register();
3530 }
3531}
3532
3533// FastEmit functions for ISD::ZERO_EXTEND.
3534
3535Register fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
3536 if (RetVT.SimpleTy != MVT::i32)
3537 return Register();
3538 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
3539}
3540
3541Register fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, Register Op0) {
3542 if (RetVT.SimpleTy != MVT::i32)
3543 return Register();
3544 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr16, RC: &X86::GR32RegClass, Op0);
3545}
3546
3547Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Register Op0) {
3548 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3549 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWYrr, RC: &X86::VR256RegClass, Op0);
3550 }
3551 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3552 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
3553 }
3554 return Register();
3555}
3556
3557Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Register Op0) {
3558 if ((Subtarget->hasAVX512())) {
3559 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZrr, RC: &X86::VR512RegClass, Op0);
3560 }
3561 return Register();
3562}
3563
3564Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(MVT RetVT, Register Op0) {
3565switch (RetVT.SimpleTy) {
3566 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
3567 case MVT::v16i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
3568 default: return Register();
3569}
3570}
3571
3572Register fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(MVT RetVT, Register Op0) {
3573 if (RetVT.SimpleTy != MVT::v32i16)
3574 return Register();
3575 if ((Subtarget->hasBWI())) {
3576 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZrr, RC: &X86::VR512RegClass, Op0);
3577 }
3578 return Register();
3579}
3580
3581Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Register Op0) {
3582 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3583 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDYrr, RC: &X86::VR256RegClass, Op0);
3584 }
3585 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3586 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
3587 }
3588 return Register();
3589}
3590
3591Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Register Op0) {
3592 if ((Subtarget->hasAVX512())) {
3593 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZrr, RC: &X86::VR512RegClass, Op0);
3594 }
3595 return Register();
3596}
3597
3598Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(MVT RetVT, Register Op0) {
3599switch (RetVT.SimpleTy) {
3600 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
3601 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
3602 default: return Register();
3603}
3604}
3605
3606Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(MVT RetVT, Register Op0) {
3607 if (RetVT.SimpleTy != MVT::v16i32)
3608 return Register();
3609 if ((Subtarget->hasAVX512())) {
3610 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZrr, RC: &X86::VR512RegClass, Op0);
3611 }
3612 return Register();
3613}
3614
3615Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(MVT RetVT, Register Op0) {
3616 if (RetVT.SimpleTy != MVT::v4i64)
3617 return Register();
3618 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3619 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQYrr, RC: &X86::VR256RegClass, Op0);
3620 }
3621 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3622 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
3623 }
3624 return Register();
3625}
3626
3627Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(MVT RetVT, Register Op0) {
3628 if (RetVT.SimpleTy != MVT::v8i64)
3629 return Register();
3630 if ((Subtarget->hasAVX512())) {
3631 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZrr, RC: &X86::VR512RegClass, Op0);
3632 }
3633 return Register();
3634}
3635
3636Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3637 switch (VT.SimpleTy) {
3638 case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0);
3639 case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0);
3640 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(RetVT, Op0);
3641 case MVT::v32i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(RetVT, Op0);
3642 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(RetVT, Op0);
3643 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(RetVT, Op0);
3644 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(RetVT, Op0);
3645 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(RetVT, Op0);
3646 default: return Register();
3647 }
3648}
3649
3650// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
3651
3652Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Register Op0) {
3653 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3654 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3655 }
3656 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3657 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3658 }
3659 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3660 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
3661 }
3662 return Register();
3663}
3664
3665Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Register Op0) {
3666 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3667 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3668 }
3669 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3670 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3671 }
3672 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3673 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
3674 }
3675 return Register();
3676}
3677
3678Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Register Op0) {
3679 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3680 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDYrr, RC: &X86::VR256RegClass, Op0);
3681 }
3682 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3683 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
3684 }
3685 return Register();
3686}
3687
3688Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Register Op0) {
3689 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3690 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3691 }
3692 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3693 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3694 }
3695 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3696 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
3697 }
3698 return Register();
3699}
3700
3701Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Register Op0) {
3702 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3703 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQYrr, RC: &X86::VR256RegClass, Op0);
3704 }
3705 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3706 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
3707 }
3708 return Register();
3709}
3710
3711Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Register Op0) {
3712 if ((Subtarget->hasAVX512())) {
3713 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZrr, RC: &X86::VR512RegClass, Op0);
3714 }
3715 return Register();
3716}
3717
3718Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, Register Op0) {
3719switch (RetVT.SimpleTy) {
3720 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
3721 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
3722 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
3723 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
3724 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
3725 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
3726 default: return Register();
3727}
3728}
3729
3730Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Register Op0) {
3731 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3732 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3733 }
3734 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3735 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3736 }
3737 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3738 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
3739 }
3740 return Register();
3741}
3742
3743Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Register Op0) {
3744 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3745 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3746 }
3747 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3748 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3749 }
3750 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3751 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
3752 }
3753 return Register();
3754}
3755
3756Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Register Op0) {
3757 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3758 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQYrr, RC: &X86::VR256RegClass, Op0);
3759 }
3760 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3761 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
3762 }
3763 return Register();
3764}
3765
3766Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, Register Op0) {
3767switch (RetVT.SimpleTy) {
3768 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
3769 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
3770 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
3771 default: return Register();
3772}
3773}
3774
3775Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, Register Op0) {
3776 if (RetVT.SimpleTy != MVT::v2i64)
3777 return Register();
3778 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3779 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3780 }
3781 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3782 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3783 }
3784 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3785 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
3786 }
3787 return Register();
3788}
3789
3790Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, Register Op0) {
3791 switch (VT.SimpleTy) {
3792 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
3793 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
3794 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
3795 default: return Register();
3796 }
3797}
3798
3799// FastEmit functions for X86ISD::CALL.
3800
3801Register fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, Register Op0) {
3802 if (RetVT.SimpleTy != MVT::isVoid)
3803 return Register();
3804 if ((!Subtarget->is64Bit())) {
3805 return fastEmitInst_r(MachineInstOpcode: X86::CALL16r, RC: &X86::GR16RegClass, Op0);
3806 }
3807 return Register();
3808}
3809
3810Register fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
3811 if (RetVT.SimpleTy != MVT::isVoid)
3812 return Register();
3813 if ((!Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3814 return fastEmitInst_r(MachineInstOpcode: X86::INDIRECT_THUNK_CALL32, RC: &X86::GR32RegClass, Op0);
3815 }
3816 if ((!Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3817 return fastEmitInst_r(MachineInstOpcode: X86::CALL32r, RC: &X86::GR32RegClass, Op0);
3818 }
3819 return Register();
3820}
3821
3822Register fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
3823 if (RetVT.SimpleTy != MVT::isVoid)
3824 return Register();
3825 if ((MF->getFunction().getParent()->getModuleFlag(Key: "import-call-optimization")) && (Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3826 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r_ImpCall, RC: &X86::GR64_ARegClass, Op0);
3827 }
3828 if ((Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3829 return fastEmitInst_r(MachineInstOpcode: X86::INDIRECT_THUNK_CALL64, RC: &X86::GR64RegClass, Op0);
3830 }
3831 if ((!MF->getFunction().getParent()->getModuleFlag(Key: "import-call-optimization")) && (Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3832 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r, RC: &X86::GR64RegClass, Op0);
3833 }
3834 return Register();
3835}
3836
3837Register fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
3838 switch (VT.SimpleTy) {
3839 case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0);
3840 case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0);
3841 case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0);
3842 default: return Register();
3843 }
3844}
3845
3846// FastEmit functions for X86ISD::CONFLICT.
3847
3848Register fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, Register Op0) {
3849 if (RetVT.SimpleTy != MVT::v4i32)
3850 return Register();
3851 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3852 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ128rr, RC: &X86::VR128XRegClass, Op0);
3853 }
3854 return Register();
3855}
3856
3857Register fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, Register Op0) {
3858 if (RetVT.SimpleTy != MVT::v8i32)
3859 return Register();
3860 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3861 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ256rr, RC: &X86::VR256XRegClass, Op0);
3862 }
3863 return Register();
3864}
3865
3866Register fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, Register Op0) {
3867 if (RetVT.SimpleTy != MVT::v16i32)
3868 return Register();
3869 if ((Subtarget->hasCDI())) {
3870 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZrr, RC: &X86::VR512RegClass, Op0);
3871 }
3872 return Register();
3873}
3874
3875Register fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, Register Op0) {
3876 if (RetVT.SimpleTy != MVT::v2i64)
3877 return Register();
3878 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3879 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ128rr, RC: &X86::VR128XRegClass, Op0);
3880 }
3881 return Register();
3882}
3883
3884Register fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, Register Op0) {
3885 if (RetVT.SimpleTy != MVT::v4i64)
3886 return Register();
3887 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3888 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ256rr, RC: &X86::VR256XRegClass, Op0);
3889 }
3890 return Register();
3891}
3892
3893Register fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, Register Op0) {
3894 if (RetVT.SimpleTy != MVT::v8i64)
3895 return Register();
3896 if ((Subtarget->hasCDI())) {
3897 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZrr, RC: &X86::VR512RegClass, Op0);
3898 }
3899 return Register();
3900}
3901
3902Register fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, Register Op0) {
3903 switch (VT.SimpleTy) {
3904 case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0);
3905 case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0);
3906 case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0);
3907 case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0);
3908 case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0);
3909 case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0);
3910 default: return Register();
3911 }
3912}
3913
3914// FastEmit functions for X86ISD::CVTNEPS2BF16.
3915
3916Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(MVT RetVT, Register Op0) {
3917 if (RetVT.SimpleTy != MVT::v8bf16)
3918 return Register();
3919 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3920 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0);
3921 }
3922 if ((Subtarget->hasAVXNECONVERT())) {
3923 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16rr, RC: &X86::VR128RegClass, Op0);
3924 }
3925 return Register();
3926}
3927
3928Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(MVT RetVT, Register Op0) {
3929 if (RetVT.SimpleTy != MVT::v8bf16)
3930 return Register();
3931 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3932 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
3933 }
3934 return Register();
3935}
3936
3937Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(MVT RetVT, Register Op0) {
3938 if (RetVT.SimpleTy != MVT::v16bf16)
3939 return Register();
3940 if ((Subtarget->hasBF16())) {
3941 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
3942 }
3943 return Register();
3944}
3945
3946Register fastEmit_X86ISD_CVTNEPS2BF16_r(MVT VT, MVT RetVT, Register Op0) {
3947 switch (VT.SimpleTy) {
3948 case MVT::v4f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(RetVT, Op0);
3949 case MVT::v8f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(RetVT, Op0);
3950 case MVT::v16f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(RetVT, Op0);
3951 default: return Register();
3952 }
3953}
3954
3955// FastEmit functions for X86ISD::CVTP2IBS.
3956
3957Register fastEmit_X86ISD_CVTP2IBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
3958 if (RetVT.SimpleTy != MVT::v8i16)
3959 return Register();
3960 if ((Subtarget->hasAVX10_2())) {
3961 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
3962 }
3963 return Register();
3964}
3965
3966Register fastEmit_X86ISD_CVTP2IBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
3967 if (RetVT.SimpleTy != MVT::v16i16)
3968 return Register();
3969 if ((Subtarget->hasAVX10_2())) {
3970 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
3971 }
3972 return Register();
3973}
3974
3975Register fastEmit_X86ISD_CVTP2IBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
3976 if (RetVT.SimpleTy != MVT::v32i16)
3977 return Register();
3978 if ((Subtarget->hasAVX10_2())) {
3979 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZrr, RC: &X86::VR512RegClass, Op0);
3980 }
3981 return Register();
3982}
3983
3984Register fastEmit_X86ISD_CVTP2IBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
3985 if (RetVT.SimpleTy != MVT::v8i16)
3986 return Register();
3987 if ((Subtarget->hasAVX10_2())) {
3988 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
3989 }
3990 return Register();
3991}
3992
3993Register fastEmit_X86ISD_CVTP2IBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
3994 if (RetVT.SimpleTy != MVT::v16i16)
3995 return Register();
3996 if ((Subtarget->hasAVX10_2())) {
3997 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
3998 }
3999 return Register();
4000}
4001
4002Register fastEmit_X86ISD_CVTP2IBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4003 if (RetVT.SimpleTy != MVT::v32i16)
4004 return Register();
4005 if ((Subtarget->hasAVX10_2())) {
4006 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZrr, RC: &X86::VR512RegClass, Op0);
4007 }
4008 return Register();
4009}
4010
4011Register fastEmit_X86ISD_CVTP2IBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4012 if (RetVT.SimpleTy != MVT::v4i32)
4013 return Register();
4014 if ((Subtarget->hasAVX10_2())) {
4015 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4016 }
4017 return Register();
4018}
4019
4020Register fastEmit_X86ISD_CVTP2IBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
4021 if (RetVT.SimpleTy != MVT::v8i32)
4022 return Register();
4023 if ((Subtarget->hasAVX10_2())) {
4024 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4025 }
4026 return Register();
4027}
4028
4029Register fastEmit_X86ISD_CVTP2IBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
4030 if (RetVT.SimpleTy != MVT::v16i32)
4031 return Register();
4032 if ((Subtarget->hasAVX10_2())) {
4033 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZrr, RC: &X86::VR512RegClass, Op0);
4034 }
4035 return Register();
4036}
4037
4038Register fastEmit_X86ISD_CVTP2IBS_r(MVT VT, MVT RetVT, Register Op0) {
4039 switch (VT.SimpleTy) {
4040 case MVT::v8f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v8f16_r(RetVT, Op0);
4041 case MVT::v16f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v16f16_r(RetVT, Op0);
4042 case MVT::v32f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v32f16_r(RetVT, Op0);
4043 case MVT::v8bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v8bf16_r(RetVT, Op0);
4044 case MVT::v16bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v16bf16_r(RetVT, Op0);
4045 case MVT::v32bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v32bf16_r(RetVT, Op0);
4046 case MVT::v4f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v4f32_r(RetVT, Op0);
4047 case MVT::v8f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v8f32_r(RetVT, Op0);
4048 case MVT::v16f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v16f32_r(RetVT, Op0);
4049 default: return Register();
4050 }
4051}
4052
4053// FastEmit functions for X86ISD::CVTP2IUBS.
4054
4055Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
4056 if (RetVT.SimpleTy != MVT::v8i16)
4057 return Register();
4058 if ((Subtarget->hasAVX10_2())) {
4059 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4060 }
4061 return Register();
4062}
4063
4064Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4065 if (RetVT.SimpleTy != MVT::v16i16)
4066 return Register();
4067 if ((Subtarget->hasAVX10_2())) {
4068 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4069 }
4070 return Register();
4071}
4072
4073Register fastEmit_X86ISD_CVTP2IUBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4074 if (RetVT.SimpleTy != MVT::v32i16)
4075 return Register();
4076 if ((Subtarget->hasAVX10_2())) {
4077 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZrr, RC: &X86::VR512RegClass, Op0);
4078 }
4079 return Register();
4080}
4081
4082Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
4083 if (RetVT.SimpleTy != MVT::v8i16)
4084 return Register();
4085 if ((Subtarget->hasAVX10_2())) {
4086 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4087 }
4088 return Register();
4089}
4090
4091Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
4092 if (RetVT.SimpleTy != MVT::v16i16)
4093 return Register();
4094 if ((Subtarget->hasAVX10_2())) {
4095 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4096 }
4097 return Register();
4098}
4099
4100Register fastEmit_X86ISD_CVTP2IUBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4101 if (RetVT.SimpleTy != MVT::v32i16)
4102 return Register();
4103 if ((Subtarget->hasAVX10_2())) {
4104 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZrr, RC: &X86::VR512RegClass, Op0);
4105 }
4106 return Register();
4107}
4108
4109Register fastEmit_X86ISD_CVTP2IUBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4110 if (RetVT.SimpleTy != MVT::v4i32)
4111 return Register();
4112 if ((Subtarget->hasAVX10_2())) {
4113 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4114 }
4115 return Register();
4116}
4117
4118Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
4119 if (RetVT.SimpleTy != MVT::v8i32)
4120 return Register();
4121 if ((Subtarget->hasAVX10_2())) {
4122 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4123 }
4124 return Register();
4125}
4126
4127Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
4128 if (RetVT.SimpleTy != MVT::v16i32)
4129 return Register();
4130 if ((Subtarget->hasAVX10_2())) {
4131 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZrr, RC: &X86::VR512RegClass, Op0);
4132 }
4133 return Register();
4134}
4135
4136Register fastEmit_X86ISD_CVTP2IUBS_r(MVT VT, MVT RetVT, Register Op0) {
4137 switch (VT.SimpleTy) {
4138 case MVT::v8f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8f16_r(RetVT, Op0);
4139 case MVT::v16f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16f16_r(RetVT, Op0);
4140 case MVT::v32f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v32f16_r(RetVT, Op0);
4141 case MVT::v8bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8bf16_r(RetVT, Op0);
4142 case MVT::v16bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16bf16_r(RetVT, Op0);
4143 case MVT::v32bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v32bf16_r(RetVT, Op0);
4144 case MVT::v4f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v4f32_r(RetVT, Op0);
4145 case MVT::v8f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8f32_r(RetVT, Op0);
4146 case MVT::v16f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16f32_r(RetVT, Op0);
4147 default: return Register();
4148 }
4149}
4150
4151// FastEmit functions for X86ISD::CVTP2SI.
4152
4153Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
4154 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4155 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
4156 }
4157 return Register();
4158}
4159
4160Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
4161 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4162 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4163 }
4164 return Register();
4165}
4166
4167Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
4168 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4169 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4170 }
4171 return Register();
4172}
4173
4174Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
4175 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4176 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4177 }
4178 return Register();
4179}
4180
4181Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
4182 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4183 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4184 }
4185 return Register();
4186}
4187
4188Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
4189 if ((Subtarget->hasFP16())) {
4190 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
4191 }
4192 return Register();
4193}
4194
4195Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4196switch (RetVT.SimpleTy) {
4197 case MVT::v8i16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
4198 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
4199 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
4200 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
4201 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
4202 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
4203 default: return Register();
4204}
4205}
4206
4207Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
4208 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
4210 }
4211 return Register();
4212}
4213
4214Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
4215 if ((Subtarget->hasFP16())) {
4216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
4217 }
4218 return Register();
4219}
4220
4221Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
4222switch (RetVT.SimpleTy) {
4223 case MVT::v16i16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
4224 case MVT::v16i32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
4225 default: return Register();
4226}
4227}
4228
4229Register fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
4230 if (RetVT.SimpleTy != MVT::v32i16)
4231 return Register();
4232 if ((Subtarget->hasFP16())) {
4233 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
4234 }
4235 return Register();
4236}
4237
4238Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
4239 if ((Subtarget->hasVLX())) {
4240 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4241 }
4242 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4243 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4244 }
4245 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4246 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4247 }
4248 return Register();
4249}
4250
4251Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
4252 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4253 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4254 }
4255 return Register();
4256}
4257
4258Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
4259 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4260 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4261 }
4262 return Register();
4263}
4264
4265Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4266switch (RetVT.SimpleTy) {
4267 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
4268 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
4269 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
4270 default: return Register();
4271}
4272}
4273
4274Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
4275 if ((Subtarget->hasVLX())) {
4276 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4277 }
4278 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4279 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
4280 }
4281 return Register();
4282}
4283
4284Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
4285 if ((Subtarget->hasDQI())) {
4286 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
4287 }
4288 return Register();
4289}
4290
4291Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
4292switch (RetVT.SimpleTy) {
4293 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
4294 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
4295 default: return Register();
4296}
4297}
4298
4299Register fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
4300 if (RetVT.SimpleTy != MVT::v16i32)
4301 return Register();
4302 if ((Subtarget->hasAVX512())) {
4303 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
4304 }
4305 return Register();
4306}
4307
4308Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
4309 if ((Subtarget->hasVLX())) {
4310 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4311 }
4312 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4313 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4314 }
4315 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4316 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4317 }
4318 return Register();
4319}
4320
4321Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
4322 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4323 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4324 }
4325 return Register();
4326}
4327
4328Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4329switch (RetVT.SimpleTy) {
4330 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
4331 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
4332 default: return Register();
4333}
4334}
4335
4336Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
4337 if ((Subtarget->hasVLX())) {
4338 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
4339 }
4340 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4341 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
4342 }
4343 return Register();
4344}
4345
4346Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
4347 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4349 }
4350 return Register();
4351}
4352
4353Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
4354switch (RetVT.SimpleTy) {
4355 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
4356 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
4357 default: return Register();
4358}
4359}
4360
4361Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
4362 if ((Subtarget->hasAVX512())) {
4363 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
4364 }
4365 return Register();
4366}
4367
4368Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
4369 if ((Subtarget->hasDQI())) {
4370 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
4371 }
4372 return Register();
4373}
4374
4375Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
4376switch (RetVT.SimpleTy) {
4377 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
4378 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
4379 default: return Register();
4380}
4381}
4382
4383Register fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
4384 switch (VT.SimpleTy) {
4385 case MVT::v8f16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(RetVT, Op0);
4386 case MVT::v16f16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(RetVT, Op0);
4387 case MVT::v32f16: return fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(RetVT, Op0);
4388 case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0);
4389 case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0);
4390 case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0);
4391 case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0);
4392 case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0);
4393 case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0);
4394 default: return Register();
4395 }
4396}
4397
4398// FastEmit functions for X86ISD::CVTP2UI.
4399
4400Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
4401 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4402 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
4403 }
4404 return Register();
4405}
4406
4407Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
4408 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4409 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4410 }
4411 return Register();
4412}
4413
4414Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
4415 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4416 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4417 }
4418 return Register();
4419}
4420
4421Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
4422 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4423 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4424 }
4425 return Register();
4426}
4427
4428Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
4429 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4430 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4431 }
4432 return Register();
4433}
4434
4435Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
4436 if ((Subtarget->hasFP16())) {
4437 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
4438 }
4439 return Register();
4440}
4441
4442Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4443switch (RetVT.SimpleTy) {
4444 case MVT::v8i16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
4445 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
4446 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
4447 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
4448 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
4449 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
4450 default: return Register();
4451}
4452}
4453
4454Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
4455 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4456 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
4457 }
4458 return Register();
4459}
4460
4461Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
4462 if ((Subtarget->hasFP16())) {
4463 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
4464 }
4465 return Register();
4466}
4467
4468Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
4469switch (RetVT.SimpleTy) {
4470 case MVT::v16i16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
4471 case MVT::v16i32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
4472 default: return Register();
4473}
4474}
4475
4476Register fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
4477 if (RetVT.SimpleTy != MVT::v32i16)
4478 return Register();
4479 if ((Subtarget->hasFP16())) {
4480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
4481 }
4482 return Register();
4483}
4484
4485Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
4486 if ((Subtarget->hasVLX())) {
4487 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4488 }
4489 return Register();
4490}
4491
4492Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
4493 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4495 }
4496 return Register();
4497}
4498
4499Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
4500 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4502 }
4503 return Register();
4504}
4505
4506Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4507switch (RetVT.SimpleTy) {
4508 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
4509 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
4510 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
4511 default: return Register();
4512}
4513}
4514
4515Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
4516 if ((Subtarget->hasVLX())) {
4517 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4518 }
4519 return Register();
4520}
4521
4522Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
4523 if ((Subtarget->hasDQI())) {
4524 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
4525 }
4526 return Register();
4527}
4528
4529Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
4530switch (RetVT.SimpleTy) {
4531 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
4532 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
4533 default: return Register();
4534}
4535}
4536
4537Register fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
4538 if (RetVT.SimpleTy != MVT::v16i32)
4539 return Register();
4540 if ((Subtarget->hasAVX512())) {
4541 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
4542 }
4543 return Register();
4544}
4545
4546Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
4547 if ((Subtarget->hasVLX())) {
4548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4549 }
4550 return Register();
4551}
4552
4553Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
4554 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4556 }
4557 return Register();
4558}
4559
4560Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4561switch (RetVT.SimpleTy) {
4562 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
4563 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
4564 default: return Register();
4565}
4566}
4567
4568Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
4569 if ((Subtarget->hasVLX())) {
4570 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
4571 }
4572 return Register();
4573}
4574
4575Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
4576 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4578 }
4579 return Register();
4580}
4581
4582Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
4583switch (RetVT.SimpleTy) {
4584 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
4585 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
4586 default: return Register();
4587}
4588}
4589
4590Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
4591 if ((Subtarget->hasAVX512())) {
4592 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
4593 }
4594 return Register();
4595}
4596
4597Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
4598 if ((Subtarget->hasDQI())) {
4599 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
4600 }
4601 return Register();
4602}
4603
4604Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
4605switch (RetVT.SimpleTy) {
4606 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
4607 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
4608 default: return Register();
4609}
4610}
4611
4612Register fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
4613 switch (VT.SimpleTy) {
4614 case MVT::v8f16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(RetVT, Op0);
4615 case MVT::v16f16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(RetVT, Op0);
4616 case MVT::v32f16: return fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(RetVT, Op0);
4617 case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0);
4618 case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0);
4619 case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0);
4620 case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0);
4621 case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0);
4622 case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0);
4623 default: return Register();
4624 }
4625}
4626
4627// FastEmit functions for X86ISD::CVTPH2PS.
4628
4629Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Register Op0) {
4630 if ((Subtarget->hasVLX())) {
4631 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4632 }
4633 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4634 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
4635 }
4636 return Register();
4637}
4638
4639Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Register Op0) {
4640 if ((Subtarget->hasVLX())) {
4641 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
4642 }
4643 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4644 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
4645 }
4646 return Register();
4647}
4648
4649Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, Register Op0) {
4650switch (RetVT.SimpleTy) {
4651 case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
4652 case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
4653 default: return Register();
4654}
4655}
4656
4657Register fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, Register Op0) {
4658 if (RetVT.SimpleTy != MVT::v16f32)
4659 return Register();
4660 if ((Subtarget->hasAVX512())) {
4661 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
4662 }
4663 return Register();
4664}
4665
4666Register fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, Register Op0) {
4667 switch (VT.SimpleTy) {
4668 case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
4669 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
4670 default: return Register();
4671 }
4672}
4673
4674// FastEmit functions for X86ISD::CVTPH2PS_SAE.
4675
4676Register fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(MVT RetVT, Register Op0) {
4677 if (RetVT.SimpleTy != MVT::v16f32)
4678 return Register();
4679 if ((Subtarget->hasAVX512())) {
4680 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrrb, RC: &X86::VR512RegClass, Op0);
4681 }
4682 return Register();
4683}
4684
4685Register fastEmit_X86ISD_CVTPH2PS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
4686 switch (VT.SimpleTy) {
4687 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(RetVT, Op0);
4688 default: return Register();
4689 }
4690}
4691
4692// FastEmit functions for X86ISD::CVTS2SI.
4693
4694Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Register Op0) {
4695 if ((Subtarget->hasFP16())) {
4696 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4697 }
4698 return Register();
4699}
4700
4701Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Register Op0) {
4702 if ((Subtarget->hasFP16())) {
4703 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4704 }
4705 return Register();
4706}
4707
4708Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4709switch (RetVT.SimpleTy) {
4710 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Op0);
4711 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Op0);
4712 default: return Register();
4713}
4714}
4715
4716Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Register Op0) {
4717 if ((Subtarget->hasAVX512())) {
4718 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4719 }
4720 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4721 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4722 }
4723 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4724 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4725 }
4726 return Register();
4727}
4728
4729Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Register Op0) {
4730 if ((Subtarget->hasAVX512())) {
4731 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4732 }
4733 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4734 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4735 }
4736 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4737 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4738 }
4739 return Register();
4740}
4741
4742Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4743switch (RetVT.SimpleTy) {
4744 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0);
4745 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0);
4746 default: return Register();
4747}
4748}
4749
4750Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Register Op0) {
4751 if ((Subtarget->hasAVX512())) {
4752 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4753 }
4754 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4755 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4756 }
4757 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4758 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4759 }
4760 return Register();
4761}
4762
4763Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Register Op0) {
4764 if ((Subtarget->hasAVX512())) {
4765 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4766 }
4767 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4768 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4769 }
4770 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4771 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4772 }
4773 return Register();
4774}
4775
4776Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4777switch (RetVT.SimpleTy) {
4778 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0);
4779 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0);
4780 default: return Register();
4781}
4782}
4783
4784Register fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, Register Op0) {
4785 switch (VT.SimpleTy) {
4786 case MVT::v8f16: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(RetVT, Op0);
4787 case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0);
4788 case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0);
4789 default: return Register();
4790 }
4791}
4792
4793// FastEmit functions for X86ISD::CVTS2UI.
4794
4795Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Register Op0) {
4796 if ((Subtarget->hasFP16())) {
4797 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4798 }
4799 return Register();
4800}
4801
4802Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Register Op0) {
4803 if ((Subtarget->hasFP16())) {
4804 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4805 }
4806 return Register();
4807}
4808
4809Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4810switch (RetVT.SimpleTy) {
4811 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Op0);
4812 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Op0);
4813 default: return Register();
4814}
4815}
4816
4817Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Register Op0) {
4818 if ((Subtarget->hasAVX512())) {
4819 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4820 }
4821 return Register();
4822}
4823
4824Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Register Op0) {
4825 if ((Subtarget->hasAVX512())) {
4826 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4827 }
4828 return Register();
4829}
4830
4831Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4832switch (RetVT.SimpleTy) {
4833 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0);
4834 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0);
4835 default: return Register();
4836}
4837}
4838
4839Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Register Op0) {
4840 if ((Subtarget->hasAVX512())) {
4841 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4842 }
4843 return Register();
4844}
4845
4846Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Register Op0) {
4847 if ((Subtarget->hasAVX512())) {
4848 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4849 }
4850 return Register();
4851}
4852
4853Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4854switch (RetVT.SimpleTy) {
4855 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0);
4856 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0);
4857 default: return Register();
4858}
4859}
4860
4861Register fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, Register Op0) {
4862 switch (VT.SimpleTy) {
4863 case MVT::v8f16: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(RetVT, Op0);
4864 case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0);
4865 case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0);
4866 default: return Register();
4867 }
4868}
4869
4870// FastEmit functions for X86ISD::CVTSI2P.
4871
4872Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
4873 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4874 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4875 }
4876 return Register();
4877}
4878
4879Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
4880 if ((Subtarget->hasVLX())) {
4881 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
4882 }
4883 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4884 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4885 }
4886 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4887 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4888 }
4889 return Register();
4890}
4891
4892Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
4893switch (RetVT.SimpleTy) {
4894 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
4895 case MVT::v2f64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
4896 default: return Register();
4897}
4898}
4899
4900Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
4901 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4902 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4903 }
4904 return Register();
4905}
4906
4907Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
4908 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4909 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4910 }
4911 return Register();
4912}
4913
4914Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
4915switch (RetVT.SimpleTy) {
4916 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
4917 case MVT::v4f32: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
4918 default: return Register();
4919}
4920}
4921
4922Register fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
4923 if (RetVT.SimpleTy != MVT::v8f16)
4924 return Register();
4925 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4926 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
4927 }
4928 return Register();
4929}
4930
4931Register fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, Register Op0) {
4932 switch (VT.SimpleTy) {
4933 case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
4934 case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
4935 case MVT::v4i64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
4936 default: return Register();
4937 }
4938}
4939
4940// FastEmit functions for X86ISD::CVTTP2IBS.
4941
4942Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
4943 if (RetVT.SimpleTy != MVT::v8i16)
4944 return Register();
4945 if ((Subtarget->hasAVX10_2())) {
4946 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4947 }
4948 return Register();
4949}
4950
4951Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4952 if (RetVT.SimpleTy != MVT::v16i16)
4953 return Register();
4954 if ((Subtarget->hasAVX10_2())) {
4955 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4956 }
4957 return Register();
4958}
4959
4960Register fastEmit_X86ISD_CVTTP2IBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4961 if (RetVT.SimpleTy != MVT::v32i16)
4962 return Register();
4963 if ((Subtarget->hasAVX10_2())) {
4964 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZrr, RC: &X86::VR512RegClass, Op0);
4965 }
4966 return Register();
4967}
4968
4969Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
4970 if (RetVT.SimpleTy != MVT::v8i16)
4971 return Register();
4972 if ((Subtarget->hasAVX10_2())) {
4973 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4974 }
4975 return Register();
4976}
4977
4978Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
4979 if (RetVT.SimpleTy != MVT::v16i16)
4980 return Register();
4981 if ((Subtarget->hasAVX10_2())) {
4982 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4983 }
4984 return Register();
4985}
4986
4987Register fastEmit_X86ISD_CVTTP2IBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4988 if (RetVT.SimpleTy != MVT::v32i16)
4989 return Register();
4990 if ((Subtarget->hasAVX10_2())) {
4991 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZrr, RC: &X86::VR512RegClass, Op0);
4992 }
4993 return Register();
4994}
4995
4996Register fastEmit_X86ISD_CVTTP2IBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4997 if (RetVT.SimpleTy != MVT::v4i32)
4998 return Register();
4999 if ((Subtarget->hasAVX10_2())) {
5000 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5001 }
5002 return Register();
5003}
5004
5005Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5006 if (RetVT.SimpleTy != MVT::v8i32)
5007 return Register();
5008 if ((Subtarget->hasAVX10_2())) {
5009 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5010 }
5011 return Register();
5012}
5013
5014Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5015 if (RetVT.SimpleTy != MVT::v16i32)
5016 return Register();
5017 if ((Subtarget->hasAVX10_2())) {
5018 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZrr, RC: &X86::VR512RegClass, Op0);
5019 }
5020 return Register();
5021}
5022
5023Register fastEmit_X86ISD_CVTTP2IBS_r(MVT VT, MVT RetVT, Register Op0) {
5024 switch (VT.SimpleTy) {
5025 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8f16_r(RetVT, Op0);
5026 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16f16_r(RetVT, Op0);
5027 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v32f16_r(RetVT, Op0);
5028 case MVT::v8bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8bf16_r(RetVT, Op0);
5029 case MVT::v16bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16bf16_r(RetVT, Op0);
5030 case MVT::v32bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v32bf16_r(RetVT, Op0);
5031 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v4f32_r(RetVT, Op0);
5032 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8f32_r(RetVT, Op0);
5033 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16f32_r(RetVT, Op0);
5034 default: return Register();
5035 }
5036}
5037
5038// FastEmit functions for X86ISD::CVTTP2IBS_SAE.
5039
5040Register fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5041 if (RetVT.SimpleTy != MVT::v32i16)
5042 return Register();
5043 if ((Subtarget->hasAVX10_2())) {
5044 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZrrb, RC: &X86::VR512RegClass, Op0);
5045 }
5046 return Register();
5047}
5048
5049Register fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5050 if (RetVT.SimpleTy != MVT::v16i32)
5051 return Register();
5052 if ((Subtarget->hasAVX10_2())) {
5053 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZrrb, RC: &X86::VR512RegClass, Op0);
5054 }
5055 return Register();
5056}
5057
5058Register fastEmit_X86ISD_CVTTP2IBS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5059 switch (VT.SimpleTy) {
5060 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v32f16_r(RetVT, Op0);
5061 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v16f32_r(RetVT, Op0);
5062 default: return Register();
5063 }
5064}
5065
5066// FastEmit functions for X86ISD::CVTTP2IUBS.
5067
5068Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
5069 if (RetVT.SimpleTy != MVT::v8i16)
5070 return Register();
5071 if ((Subtarget->hasAVX10_2())) {
5072 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5073 }
5074 return Register();
5075}
5076
5077Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
5078 if (RetVT.SimpleTy != MVT::v16i16)
5079 return Register();
5080 if ((Subtarget->hasAVX10_2())) {
5081 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5082 }
5083 return Register();
5084}
5085
5086Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
5087 if (RetVT.SimpleTy != MVT::v32i16)
5088 return Register();
5089 if ((Subtarget->hasAVX10_2())) {
5090 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZrr, RC: &X86::VR512RegClass, Op0);
5091 }
5092 return Register();
5093}
5094
5095Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
5096 if (RetVT.SimpleTy != MVT::v8i16)
5097 return Register();
5098 if ((Subtarget->hasAVX10_2())) {
5099 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5100 }
5101 return Register();
5102}
5103
5104Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
5105 if (RetVT.SimpleTy != MVT::v16i16)
5106 return Register();
5107 if ((Subtarget->hasAVX10_2())) {
5108 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5109 }
5110 return Register();
5111}
5112
5113Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
5114 if (RetVT.SimpleTy != MVT::v32i16)
5115 return Register();
5116 if ((Subtarget->hasAVX10_2())) {
5117 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZrr, RC: &X86::VR512RegClass, Op0);
5118 }
5119 return Register();
5120}
5121
5122Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5123 if (RetVT.SimpleTy != MVT::v4i32)
5124 return Register();
5125 if ((Subtarget->hasAVX10_2())) {
5126 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5127 }
5128 return Register();
5129}
5130
5131Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5132 if (RetVT.SimpleTy != MVT::v8i32)
5133 return Register();
5134 if ((Subtarget->hasAVX10_2())) {
5135 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5136 }
5137 return Register();
5138}
5139
5140Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5141 if (RetVT.SimpleTy != MVT::v16i32)
5142 return Register();
5143 if ((Subtarget->hasAVX10_2())) {
5144 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZrr, RC: &X86::VR512RegClass, Op0);
5145 }
5146 return Register();
5147}
5148
5149Register fastEmit_X86ISD_CVTTP2IUBS_r(MVT VT, MVT RetVT, Register Op0) {
5150 switch (VT.SimpleTy) {
5151 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f16_r(RetVT, Op0);
5152 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f16_r(RetVT, Op0);
5153 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v32f16_r(RetVT, Op0);
5154 case MVT::v8bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8bf16_r(RetVT, Op0);
5155 case MVT::v16bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16bf16_r(RetVT, Op0);
5156 case MVT::v32bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v32bf16_r(RetVT, Op0);
5157 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v4f32_r(RetVT, Op0);
5158 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f32_r(RetVT, Op0);
5159 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f32_r(RetVT, Op0);
5160 default: return Register();
5161 }
5162}
5163
5164// FastEmit functions for X86ISD::CVTTP2IUBS_SAE.
5165
5166Register fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5167 if (RetVT.SimpleTy != MVT::v32i16)
5168 return Register();
5169 if ((Subtarget->hasAVX10_2())) {
5170 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZrrb, RC: &X86::VR512RegClass, Op0);
5171 }
5172 return Register();
5173}
5174
5175Register fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5176 if (RetVT.SimpleTy != MVT::v16i32)
5177 return Register();
5178 if ((Subtarget->hasAVX10_2())) {
5179 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZrrb, RC: &X86::VR512RegClass, Op0);
5180 }
5181 return Register();
5182}
5183
5184Register fastEmit_X86ISD_CVTTP2IUBS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5185 switch (VT.SimpleTy) {
5186 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v32f16_r(RetVT, Op0);
5187 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v16f32_r(RetVT, Op0);
5188 default: return Register();
5189 }
5190}
5191
5192// FastEmit functions for X86ISD::CVTTP2SI.
5193
5194Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
5195 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5196 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
5197 }
5198 return Register();
5199}
5200
5201Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
5202 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5203 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5204 }
5205 return Register();
5206}
5207
5208Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
5209 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5210 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
5211 }
5212 return Register();
5213}
5214
5215Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
5216 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5217 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5218 }
5219 return Register();
5220}
5221
5222Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
5223 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5224 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5225 }
5226 return Register();
5227}
5228
5229Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
5230 if ((Subtarget->hasFP16())) {
5231 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
5232 }
5233 return Register();
5234}
5235
5236Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
5237switch (RetVT.SimpleTy) {
5238 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
5239 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
5240 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
5241 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
5242 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
5243 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
5244 default: return Register();
5245}
5246}
5247
5248Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
5249 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5250 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
5251 }
5252 return Register();
5253}
5254
5255Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
5256 if ((Subtarget->hasFP16())) {
5257 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
5258 }
5259 return Register();
5260}
5261
5262Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
5263switch (RetVT.SimpleTy) {
5264 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
5265 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
5266 default: return Register();
5267}
5268}
5269
5270Register fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
5271 if (RetVT.SimpleTy != MVT::v32i16)
5272 return Register();
5273 if ((Subtarget->hasFP16())) {
5274 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
5275 }
5276 return Register();
5277}
5278
5279Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5280 if ((Subtarget->hasVLX())) {
5281 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5282 }
5283 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5284 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
5285 }
5286 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5287 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
5288 }
5289 return Register();
5290}
5291
5292Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5293 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5294 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5295 }
5296 return Register();
5297}
5298
5299Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5300 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5301 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5302 }
5303 return Register();
5304}
5305
5306Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
5307switch (RetVT.SimpleTy) {
5308 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
5309 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
5310 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
5311 default: return Register();
5312}
5313}
5314
5315Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5316 if ((Subtarget->hasVLX())) {
5317 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
5318 }
5319 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5320 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
5321 }
5322 return Register();
5323}
5324
5325Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5326 if ((Subtarget->hasDQI())) {
5327 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
5328 }
5329 return Register();
5330}
5331
5332Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
5333switch (RetVT.SimpleTy) {
5334 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
5335 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
5336 default: return Register();
5337}
5338}
5339
5340Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
5341 if (RetVT.SimpleTy != MVT::v16i32)
5342 return Register();
5343 if ((Subtarget->hasAVX512())) {
5344 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
5345 }
5346 return Register();
5347}
5348
5349Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5350 if ((Subtarget->hasVLX())) {
5351 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5352 }
5353 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5354 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
5355 }
5356 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
5358 }
5359 return Register();
5360}
5361
5362Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5363 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5364 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5365 }
5366 return Register();
5367}
5368
5369Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
5370switch (RetVT.SimpleTy) {
5371 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
5372 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
5373 default: return Register();
5374}
5375}
5376
5377Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5378 if ((Subtarget->hasVLX())) {
5379 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
5380 }
5381 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5382 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
5383 }
5384 return Register();
5385}
5386
5387Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5388 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5389 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5390 }
5391 return Register();
5392}
5393
5394Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
5395switch (RetVT.SimpleTy) {
5396 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
5397 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
5398 default: return Register();
5399}
5400}
5401
5402Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5403 if ((Subtarget->hasAVX512())) {
5404 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
5405 }
5406 return Register();
5407}
5408
5409Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5410 if ((Subtarget->hasDQI())) {
5411 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
5412 }
5413 return Register();
5414}
5415
5416Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
5417switch (RetVT.SimpleTy) {
5418 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
5419 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
5420 default: return Register();
5421}
5422}
5423
5424Register fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
5425 switch (VT.SimpleTy) {
5426 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
5427 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
5428 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
5429 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
5430 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
5431 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
5432 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
5433 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
5434 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
5435 default: return Register();
5436 }
5437}
5438
5439// FastEmit functions for X86ISD::CVTTP2SIS.
5440
5441Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5442 if ((Subtarget->hasAVX10_2())) {
5443 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5444 }
5445 return Register();
5446}
5447
5448Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5449 if ((Subtarget->hasAVX10_2())) {
5450 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5451 }
5452 return Register();
5453}
5454
5455Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5456 if ((Subtarget->hasAVX10_2())) {
5457 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5458 }
5459 return Register();
5460}
5461
5462Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5463switch (RetVT.SimpleTy) {
5464 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i32_r(Op0);
5465 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v2i64_r(Op0);
5466 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i64_r(Op0);
5467 default: return Register();
5468}
5469}
5470
5471Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5472 if ((Subtarget->hasAVX10_2())) {
5473 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5474 }
5475 return Register();
5476}
5477
5478Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5479 if ((Subtarget->hasAVX10_2())) {
5480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZrr, RC: &X86::VR512RegClass, Op0);
5481 }
5482 return Register();
5483}
5484
5485Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5486switch (RetVT.SimpleTy) {
5487 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i32_r(Op0);
5488 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i64_r(Op0);
5489 default: return Register();
5490}
5491}
5492
5493Register fastEmit_X86ISD_CVTTP2SIS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5494 if (RetVT.SimpleTy != MVT::v16i32)
5495 return Register();
5496 if ((Subtarget->hasAVX10_2())) {
5497 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZrr, RC: &X86::VR512RegClass, Op0);
5498 }
5499 return Register();
5500}
5501
5502Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5503 if ((Subtarget->hasAVX10_2())) {
5504 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5505 }
5506 return Register();
5507}
5508
5509Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5510 if ((Subtarget->hasAVX10_2())) {
5511 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5512 }
5513 return Register();
5514}
5515
5516Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
5517switch (RetVT.SimpleTy) {
5518 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v4i32_r(Op0);
5519 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v2i64_r(Op0);
5520 default: return Register();
5521}
5522}
5523
5524Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5525 if ((Subtarget->hasAVX10_2())) {
5526 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ256rr, RC: &X86::VR128XRegClass, Op0);
5527 }
5528 return Register();
5529}
5530
5531Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5532 if ((Subtarget->hasAVX10_2())) {
5533 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5534 }
5535 return Register();
5536}
5537
5538Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_r(MVT RetVT, Register Op0) {
5539switch (RetVT.SimpleTy) {
5540 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i32_r(Op0);
5541 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i64_r(Op0);
5542 default: return Register();
5543}
5544}
5545
5546Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5547 if ((Subtarget->hasAVX10_2())) {
5548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZrr, RC: &X86::VR256XRegClass, Op0);
5549 }
5550 return Register();
5551}
5552
5553Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5554 if ((Subtarget->hasAVX10_2())) {
5555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZrr, RC: &X86::VR512RegClass, Op0);
5556 }
5557 return Register();
5558}
5559
5560Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_r(MVT RetVT, Register Op0) {
5561switch (RetVT.SimpleTy) {
5562 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i32_r(Op0);
5563 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i64_r(Op0);
5564 default: return Register();
5565}
5566}
5567
5568Register fastEmit_X86ISD_CVTTP2SIS_r(MVT VT, MVT RetVT, Register Op0) {
5569 switch (VT.SimpleTy) {
5570 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_r(RetVT, Op0);
5571 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_r(RetVT, Op0);
5572 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v16f32_r(RetVT, Op0);
5573 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_r(RetVT, Op0);
5574 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_r(RetVT, Op0);
5575 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_r(RetVT, Op0);
5576 default: return Register();
5577 }
5578}
5579
5580// FastEmit functions for X86ISD::CVTTP2SIS_SAE.
5581
5582Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
5583 if (RetVT.SimpleTy != MVT::v4i64)
5584 return Register();
5585 if ((Subtarget->hasAVX10_2())) {
5586 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
5587 }
5588 return Register();
5589}
5590
5591Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
5592 if (RetVT.SimpleTy != MVT::v8i64)
5593 return Register();
5594 if ((Subtarget->hasAVX10_2())) {
5595 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZrrb, RC: &X86::VR512RegClass, Op0);
5596 }
5597 return Register();
5598}
5599
5600Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5601 if (RetVT.SimpleTy != MVT::v16i32)
5602 return Register();
5603 if ((Subtarget->hasAVX10_2())) {
5604 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZrrb, RC: &X86::VR512RegClass, Op0);
5605 }
5606 return Register();
5607}
5608
5609Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5610 if ((Subtarget->hasAVX10_2())) {
5611 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ256rrb, RC: &X86::VR128XRegClass, Op0);
5612 }
5613 return Register();
5614}
5615
5616Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5617 if ((Subtarget->hasAVX10_2())) {
5618 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
5619 }
5620 return Register();
5621}
5622
5623Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_r(MVT RetVT, Register Op0) {
5624switch (RetVT.SimpleTy) {
5625 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i32_r(Op0);
5626 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i64_r(Op0);
5627 default: return Register();
5628}
5629}
5630
5631Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5632 if ((Subtarget->hasAVX10_2())) {
5633 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZrrb, RC: &X86::VR256XRegClass, Op0);
5634 }
5635 return Register();
5636}
5637
5638Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5639 if ((Subtarget->hasAVX10_2())) {
5640 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZrrb, RC: &X86::VR512RegClass, Op0);
5641 }
5642 return Register();
5643}
5644
5645Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
5646switch (RetVT.SimpleTy) {
5647 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5648 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5649 default: return Register();
5650}
5651}
5652
5653Register fastEmit_X86ISD_CVTTP2SIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5654 switch (VT.SimpleTy) {
5655 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f32_r(RetVT, Op0);
5656 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f32_r(RetVT, Op0);
5657 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v16f32_r(RetVT, Op0);
5658 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_r(RetVT, Op0);
5659 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_r(RetVT, Op0);
5660 default: return Register();
5661 }
5662}
5663
5664// FastEmit functions for X86ISD::CVTTP2SI_SAE.
5665
5666Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
5667 if (RetVT.SimpleTy != MVT::v8i64)
5668 return Register();
5669 if ((Subtarget->hasFP16())) {
5670 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrrb, RC: &X86::VR512RegClass, Op0);
5671 }
5672 return Register();
5673}
5674
5675Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
5676 if (RetVT.SimpleTy != MVT::v16i32)
5677 return Register();
5678 if ((Subtarget->hasFP16())) {
5679 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrrb, RC: &X86::VR512RegClass, Op0);
5680 }
5681 return Register();
5682}
5683
5684Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5685 if (RetVT.SimpleTy != MVT::v32i16)
5686 return Register();
5687 if ((Subtarget->hasFP16())) {
5688 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrrb, RC: &X86::VR512RegClass, Op0);
5689 }
5690 return Register();
5691}
5692
5693Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
5694 if (RetVT.SimpleTy != MVT::v8i64)
5695 return Register();
5696 if ((Subtarget->hasDQI())) {
5697 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrrb, RC: &X86::VR512RegClass, Op0);
5698 }
5699 return Register();
5700}
5701
5702Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5703 if (RetVT.SimpleTy != MVT::v16i32)
5704 return Register();
5705 if ((Subtarget->hasAVX512())) {
5706 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrrb, RC: &X86::VR512RegClass, Op0);
5707 }
5708 return Register();
5709}
5710
5711Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5712 if ((Subtarget->hasAVX512())) {
5713 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrrb, RC: &X86::VR256XRegClass, Op0);
5714 }
5715 return Register();
5716}
5717
5718Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5719 if ((Subtarget->hasDQI())) {
5720 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrrb, RC: &X86::VR512RegClass, Op0);
5721 }
5722 return Register();
5723}
5724
5725Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
5726switch (RetVT.SimpleTy) {
5727 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5728 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5729 default: return Register();
5730}
5731}
5732
5733Register fastEmit_X86ISD_CVTTP2SI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5734 switch (VT.SimpleTy) {
5735 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(RetVT, Op0);
5736 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(RetVT, Op0);
5737 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(RetVT, Op0);
5738 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(RetVT, Op0);
5739 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(RetVT, Op0);
5740 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(RetVT, Op0);
5741 default: return Register();
5742 }
5743}
5744
5745// FastEmit functions for X86ISD::CVTTP2UI.
5746
5747Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
5748 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5749 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
5750 }
5751 return Register();
5752}
5753
5754Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
5755 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5756 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5757 }
5758 return Register();
5759}
5760
5761Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
5762 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5763 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5764 }
5765 return Register();
5766}
5767
5768Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
5769 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5770 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5771 }
5772 return Register();
5773}
5774
5775Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
5776 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5777 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5778 }
5779 return Register();
5780}
5781
5782Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
5783 if ((Subtarget->hasFP16())) {
5784 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
5785 }
5786 return Register();
5787}
5788
5789Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
5790switch (RetVT.SimpleTy) {
5791 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
5792 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
5793 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
5794 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
5795 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
5796 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
5797 default: return Register();
5798}
5799}
5800
5801Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
5802 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5803 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
5804 }
5805 return Register();
5806}
5807
5808Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
5809 if ((Subtarget->hasFP16())) {
5810 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
5811 }
5812 return Register();
5813}
5814
5815Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
5816switch (RetVT.SimpleTy) {
5817 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
5818 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
5819 default: return Register();
5820}
5821}
5822
5823Register fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
5824 if (RetVT.SimpleTy != MVT::v32i16)
5825 return Register();
5826 if ((Subtarget->hasFP16())) {
5827 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
5828 }
5829 return Register();
5830}
5831
5832Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5833 if ((Subtarget->hasVLX())) {
5834 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5835 }
5836 return Register();
5837}
5838
5839Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5840 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5841 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5842 }
5843 return Register();
5844}
5845
5846Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5847 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5848 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5849 }
5850 return Register();
5851}
5852
5853Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
5854switch (RetVT.SimpleTy) {
5855 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
5856 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
5857 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
5858 default: return Register();
5859}
5860}
5861
5862Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5863 if ((Subtarget->hasVLX())) {
5864 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5865 }
5866 return Register();
5867}
5868
5869Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5870 if ((Subtarget->hasDQI())) {
5871 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
5872 }
5873 return Register();
5874}
5875
5876Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
5877switch (RetVT.SimpleTy) {
5878 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
5879 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
5880 default: return Register();
5881}
5882}
5883
5884Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
5885 if (RetVT.SimpleTy != MVT::v16i32)
5886 return Register();
5887 if ((Subtarget->hasAVX512())) {
5888 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
5889 }
5890 return Register();
5891}
5892
5893Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5894 if ((Subtarget->hasVLX())) {
5895 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5896 }
5897 return Register();
5898}
5899
5900Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5901 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5902 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5903 }
5904 return Register();
5905}
5906
5907Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
5908switch (RetVT.SimpleTy) {
5909 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
5910 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
5911 default: return Register();
5912}
5913}
5914
5915Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5916 if ((Subtarget->hasVLX())) {
5917 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
5918 }
5919 return Register();
5920}
5921
5922Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5923 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5924 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5925 }
5926 return Register();
5927}
5928
5929Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
5930switch (RetVT.SimpleTy) {
5931 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
5932 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
5933 default: return Register();
5934}
5935}
5936
5937Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5938 if ((Subtarget->hasAVX512())) {
5939 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
5940 }
5941 return Register();
5942}
5943
5944Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5945 if ((Subtarget->hasDQI())) {
5946 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
5947 }
5948 return Register();
5949}
5950
5951Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
5952switch (RetVT.SimpleTy) {
5953 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
5954 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
5955 default: return Register();
5956}
5957}
5958
5959Register fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
5960 switch (VT.SimpleTy) {
5961 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
5962 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
5963 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
5964 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
5965 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
5966 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
5967 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
5968 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
5969 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
5970 default: return Register();
5971 }
5972}
5973
5974// FastEmit functions for X86ISD::CVTTP2UIS.
5975
5976Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5977 if ((Subtarget->hasAVX10_2())) {
5978 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5979 }
5980 return Register();
5981}
5982
5983Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5984 if ((Subtarget->hasAVX10_2())) {
5985 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5986 }
5987 return Register();
5988}
5989
5990Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5991 if ((Subtarget->hasAVX10_2())) {
5992 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5993 }
5994 return Register();
5995}
5996
5997Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5998switch (RetVT.SimpleTy) {
5999 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i32_r(Op0);
6000 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v2i64_r(Op0);
6001 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i64_r(Op0);
6002 default: return Register();
6003}
6004}
6005
6006Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i32_r(Register Op0) {
6007 if ((Subtarget->hasAVX10_2())) {
6008 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZ256rr, RC: &X86::VR256XRegClass, Op0);
6009 }
6010 return Register();
6011}
6012
6013Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i64_r(Register Op0) {
6014 if ((Subtarget->hasAVX10_2())) {
6015 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZrr, RC: &X86::VR512RegClass, Op0);
6016 }
6017 return Register();
6018}
6019
6020Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_r(MVT RetVT, Register Op0) {
6021switch (RetVT.SimpleTy) {
6022 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i32_r(Op0);
6023 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i64_r(Op0);
6024 default: return Register();
6025}
6026}
6027
6028Register fastEmit_X86ISD_CVTTP2UIS_MVT_v16f32_r(MVT RetVT, Register Op0) {
6029 if (RetVT.SimpleTy != MVT::v16i32)
6030 return Register();
6031 if ((Subtarget->hasAVX10_2())) {
6032 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZrr, RC: &X86::VR512RegClass, Op0);
6033 }
6034 return Register();
6035}
6036
6037Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v4i32_r(Register Op0) {
6038 if ((Subtarget->hasAVX10_2())) {
6039 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6040 }
6041 return Register();
6042}
6043
6044Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v2i64_r(Register Op0) {
6045 if ((Subtarget->hasAVX10_2())) {
6046 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6047 }
6048 return Register();
6049}
6050
6051Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6052switch (RetVT.SimpleTy) {
6053 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v4i32_r(Op0);
6054 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v2i64_r(Op0);
6055 default: return Register();
6056}
6057}
6058
6059Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i32_r(Register Op0) {
6060 if ((Subtarget->hasAVX10_2())) {
6061 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ256rr, RC: &X86::VR128XRegClass, Op0);
6062 }
6063 return Register();
6064}
6065
6066Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i64_r(Register Op0) {
6067 if ((Subtarget->hasAVX10_2())) {
6068 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ256rr, RC: &X86::VR256XRegClass, Op0);
6069 }
6070 return Register();
6071}
6072
6073Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_r(MVT RetVT, Register Op0) {
6074switch (RetVT.SimpleTy) {
6075 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i32_r(Op0);
6076 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i64_r(Op0);
6077 default: return Register();
6078}
6079}
6080
6081Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6082 if ((Subtarget->hasAVX10_2())) {
6083 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZrr, RC: &X86::VR256XRegClass, Op0);
6084 }
6085 return Register();
6086}
6087
6088Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6089 if ((Subtarget->hasAVX10_2())) {
6090 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZrr, RC: &X86::VR512RegClass, Op0);
6091 }
6092 return Register();
6093}
6094
6095Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_r(MVT RetVT, Register Op0) {
6096switch (RetVT.SimpleTy) {
6097 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i32_r(Op0);
6098 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i64_r(Op0);
6099 default: return Register();
6100}
6101}
6102
6103Register fastEmit_X86ISD_CVTTP2UIS_r(MVT VT, MVT RetVT, Register Op0) {
6104 switch (VT.SimpleTy) {
6105 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_r(RetVT, Op0);
6106 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_r(RetVT, Op0);
6107 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v16f32_r(RetVT, Op0);
6108 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_r(RetVT, Op0);
6109 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_r(RetVT, Op0);
6110 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_r(RetVT, Op0);
6111 default: return Register();
6112 }
6113}
6114
6115// FastEmit functions for X86ISD::CVTTP2UIS_SAE.
6116
6117Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6118 if (RetVT.SimpleTy != MVT::v4i64)
6119 return Register();
6120 if ((Subtarget->hasAVX10_2())) {
6121 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
6122 }
6123 return Register();
6124}
6125
6126Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
6127 if (RetVT.SimpleTy != MVT::v8i64)
6128 return Register();
6129 if ((Subtarget->hasAVX10_2())) {
6130 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZrrb, RC: &X86::VR512RegClass, Op0);
6131 }
6132 return Register();
6133}
6134
6135Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6136 if (RetVT.SimpleTy != MVT::v16i32)
6137 return Register();
6138 if ((Subtarget->hasAVX10_2())) {
6139 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZrrb, RC: &X86::VR512RegClass, Op0);
6140 }
6141 return Register();
6142}
6143
6144Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i32_r(Register Op0) {
6145 if ((Subtarget->hasAVX10_2())) {
6146 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ256rrb, RC: &X86::VR128XRegClass, Op0);
6147 }
6148 return Register();
6149}
6150
6151Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i64_r(Register Op0) {
6152 if ((Subtarget->hasAVX10_2())) {
6153 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
6154 }
6155 return Register();
6156}
6157
6158Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_r(MVT RetVT, Register Op0) {
6159switch (RetVT.SimpleTy) {
6160 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i32_r(Op0);
6161 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i64_r(Op0);
6162 default: return Register();
6163}
6164}
6165
6166Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6167 if ((Subtarget->hasAVX10_2())) {
6168 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZrrb, RC: &X86::VR256XRegClass, Op0);
6169 }
6170 return Register();
6171}
6172
6173Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6174 if ((Subtarget->hasAVX10_2())) {
6175 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZrrb, RC: &X86::VR512RegClass, Op0);
6176 }
6177 return Register();
6178}
6179
6180Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6181switch (RetVT.SimpleTy) {
6182 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
6183 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
6184 default: return Register();
6185}
6186}
6187
6188Register fastEmit_X86ISD_CVTTP2UIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6189 switch (VT.SimpleTy) {
6190 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f32_r(RetVT, Op0);
6191 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f32_r(RetVT, Op0);
6192 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v16f32_r(RetVT, Op0);
6193 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_r(RetVT, Op0);
6194 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_r(RetVT, Op0);
6195 default: return Register();
6196 }
6197}
6198
6199// FastEmit functions for X86ISD::CVTTP2UI_SAE.
6200
6201Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6202 if (RetVT.SimpleTy != MVT::v8i64)
6203 return Register();
6204 if ((Subtarget->hasFP16())) {
6205 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6206 }
6207 return Register();
6208}
6209
6210Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
6211 if (RetVT.SimpleTy != MVT::v16i32)
6212 return Register();
6213 if ((Subtarget->hasFP16())) {
6214 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrrb, RC: &X86::VR512RegClass, Op0);
6215 }
6216 return Register();
6217}
6218
6219Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
6220 if (RetVT.SimpleTy != MVT::v32i16)
6221 return Register();
6222 if ((Subtarget->hasFP16())) {
6223 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrrb, RC: &X86::VR512RegClass, Op0);
6224 }
6225 return Register();
6226}
6227
6228Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
6229 if (RetVT.SimpleTy != MVT::v8i64)
6230 return Register();
6231 if ((Subtarget->hasDQI())) {
6232 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6233 }
6234 return Register();
6235}
6236
6237Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6238 if (RetVT.SimpleTy != MVT::v16i32)
6239 return Register();
6240 if ((Subtarget->hasAVX512())) {
6241 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrrb, RC: &X86::VR512RegClass, Op0);
6242 }
6243 return Register();
6244}
6245
6246Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6247 if ((Subtarget->hasAVX512())) {
6248 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrrb, RC: &X86::VR256XRegClass, Op0);
6249 }
6250 return Register();
6251}
6252
6253Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6254 if ((Subtarget->hasDQI())) {
6255 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6256 }
6257 return Register();
6258}
6259
6260Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6261switch (RetVT.SimpleTy) {
6262 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
6263 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
6264 default: return Register();
6265}
6266}
6267
6268Register fastEmit_X86ISD_CVTTP2UI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6269 switch (VT.SimpleTy) {
6270 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(RetVT, Op0);
6271 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(RetVT, Op0);
6272 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(RetVT, Op0);
6273 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(RetVT, Op0);
6274 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(RetVT, Op0);
6275 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(RetVT, Op0);
6276 default: return Register();
6277 }
6278}
6279
6280// FastEmit functions for X86ISD::CVTTS2SI.
6281
6282Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Register Op0) {
6283 if ((Subtarget->hasFP16())) {
6284 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6285 }
6286 return Register();
6287}
6288
6289Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Register Op0) {
6290 if ((Subtarget->hasFP16())) {
6291 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6292 }
6293 return Register();
6294}
6295
6296Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
6297switch (RetVT.SimpleTy) {
6298 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Op0);
6299 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Op0);
6300 default: return Register();
6301}
6302}
6303
6304Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Register Op0) {
6305 if ((Subtarget->hasAVX512())) {
6306 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6307 }
6308 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6309 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6310 }
6311 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6312 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6313 }
6314 return Register();
6315}
6316
6317Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Register Op0) {
6318 if ((Subtarget->hasAVX512())) {
6319 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6320 }
6321 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6322 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6323 }
6324 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6325 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6326 }
6327 return Register();
6328}
6329
6330Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
6331switch (RetVT.SimpleTy) {
6332 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0);
6333 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0);
6334 default: return Register();
6335}
6336}
6337
6338Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Register Op0) {
6339 if ((Subtarget->hasAVX512())) {
6340 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6341 }
6342 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6343 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6344 }
6345 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6346 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6347 }
6348 return Register();
6349}
6350
6351Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Register Op0) {
6352 if ((Subtarget->hasAVX512())) {
6353 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6354 }
6355 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6356 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6357 }
6358 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6359 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6360 }
6361 return Register();
6362}
6363
6364Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
6365switch (RetVT.SimpleTy) {
6366 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0);
6367 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0);
6368 default: return Register();
6369}
6370}
6371
6372Register fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, Register Op0) {
6373 switch (VT.SimpleTy) {
6374 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(RetVT, Op0);
6375 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0);
6376 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0);
6377 default: return Register();
6378 }
6379}
6380
6381// FastEmit functions for X86ISD::CVTTS2SIS.
6382
6383Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i32_r(Register Op0) {
6384 if ((Subtarget->hasAVX10_2())) {
6385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SISrr_Int, RC: &X86::GR32RegClass, Op0);
6386 }
6387 return Register();
6388}
6389
6390Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i64_r(Register Op0) {
6391 if ((Subtarget->hasAVX10_2())) {
6392 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6393 }
6394 return Register();
6395}
6396
6397Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6398switch (RetVT.SimpleTy) {
6399 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i32_r(Op0);
6400 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i64_r(Op0);
6401 default: return Register();
6402}
6403}
6404
6405Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i32_r(Register Op0) {
6406 if ((Subtarget->hasAVX10_2())) {
6407 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SISrr_Int, RC: &X86::GR32RegClass, Op0);
6408 }
6409 return Register();
6410}
6411
6412Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i64_r(Register Op0) {
6413 if ((Subtarget->hasAVX10_2())) {
6414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6415 }
6416 return Register();
6417}
6418
6419Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6420switch (RetVT.SimpleTy) {
6421 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i32_r(Op0);
6422 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i64_r(Op0);
6423 default: return Register();
6424}
6425}
6426
6427Register fastEmit_X86ISD_CVTTS2SIS_r(MVT VT, MVT RetVT, Register Op0) {
6428 switch (VT.SimpleTy) {
6429 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_r(RetVT, Op0);
6430 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_r(RetVT, Op0);
6431 default: return Register();
6432 }
6433}
6434
6435// FastEmit functions for X86ISD::CVTTS2SIS_SAE.
6436
6437Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6438 if ((Subtarget->hasAVX10_2())) {
6439 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SISrrb_Int, RC: &X86::GR32RegClass, Op0);
6440 }
6441 return Register();
6442}
6443
6444Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6445 if ((Subtarget->hasAVX10_2())) {
6446 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6447 }
6448 return Register();
6449}
6450
6451Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6452switch (RetVT.SimpleTy) {
6453 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i32_r(Op0);
6454 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i64_r(Op0);
6455 default: return Register();
6456}
6457}
6458
6459Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6460 if ((Subtarget->hasAVX10_2())) {
6461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SISrrb_Int, RC: &X86::GR32RegClass, Op0);
6462 }
6463 return Register();
6464}
6465
6466Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6467 if ((Subtarget->hasAVX10_2())) {
6468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6469 }
6470 return Register();
6471}
6472
6473Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6474switch (RetVT.SimpleTy) {
6475 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i32_r(Op0);
6476 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i64_r(Op0);
6477 default: return Register();
6478}
6479}
6480
6481Register fastEmit_X86ISD_CVTTS2SIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6482 switch (VT.SimpleTy) {
6483 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_r(RetVT, Op0);
6484 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_r(RetVT, Op0);
6485 default: return Register();
6486 }
6487}
6488
6489// FastEmit functions for X86ISD::CVTTS2SI_SAE.
6490
6491Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Register Op0) {
6492 if ((Subtarget->hasFP16())) {
6493 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6494 }
6495 return Register();
6496}
6497
6498Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Register Op0) {
6499 if ((Subtarget->hasFP16())) {
6500 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6501 }
6502 return Register();
6503}
6504
6505Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6506switch (RetVT.SimpleTy) {
6507 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Op0);
6508 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Op0);
6509 default: return Register();
6510}
6511}
6512
6513Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6514 if ((Subtarget->hasAVX512())) {
6515 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6516 }
6517 return Register();
6518}
6519
6520Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6521 if ((Subtarget->hasAVX512())) {
6522 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6523 }
6524 return Register();
6525}
6526
6527Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6528switch (RetVT.SimpleTy) {
6529 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Op0);
6530 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Op0);
6531 default: return Register();
6532}
6533}
6534
6535Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6536 if ((Subtarget->hasAVX512())) {
6537 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6538 }
6539 return Register();
6540}
6541
6542Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6543 if ((Subtarget->hasAVX512())) {
6544 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6545 }
6546 return Register();
6547}
6548
6549Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6550switch (RetVT.SimpleTy) {
6551 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Op0);
6552 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Op0);
6553 default: return Register();
6554}
6555}
6556
6557Register fastEmit_X86ISD_CVTTS2SI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6558 switch (VT.SimpleTy) {
6559 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(RetVT, Op0);
6560 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(RetVT, Op0);
6561 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(RetVT, Op0);
6562 default: return Register();
6563 }
6564}
6565
6566// FastEmit functions for X86ISD::CVTTS2UI.
6567
6568Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Register Op0) {
6569 if ((Subtarget->hasFP16())) {
6570 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6571 }
6572 return Register();
6573}
6574
6575Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Register Op0) {
6576 if ((Subtarget->hasFP16())) {
6577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6578 }
6579 return Register();
6580}
6581
6582Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
6583switch (RetVT.SimpleTy) {
6584 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Op0);
6585 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Op0);
6586 default: return Register();
6587}
6588}
6589
6590Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Register Op0) {
6591 if ((Subtarget->hasAVX512())) {
6592 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6593 }
6594 return Register();
6595}
6596
6597Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Register Op0) {
6598 if ((Subtarget->hasAVX512())) {
6599 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6600 }
6601 return Register();
6602}
6603
6604Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
6605switch (RetVT.SimpleTy) {
6606 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0);
6607 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0);
6608 default: return Register();
6609}
6610}
6611
6612Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Register Op0) {
6613 if ((Subtarget->hasAVX512())) {
6614 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6615 }
6616 return Register();
6617}
6618
6619Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Register Op0) {
6620 if ((Subtarget->hasAVX512())) {
6621 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6622 }
6623 return Register();
6624}
6625
6626Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
6627switch (RetVT.SimpleTy) {
6628 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0);
6629 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0);
6630 default: return Register();
6631}
6632}
6633
6634Register fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, Register Op0) {
6635 switch (VT.SimpleTy) {
6636 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(RetVT, Op0);
6637 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0);
6638 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0);
6639 default: return Register();
6640 }
6641}
6642
6643// FastEmit functions for X86ISD::CVTTS2UIS.
6644
6645Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i32_r(Register Op0) {
6646 if ((Subtarget->hasAVX10_2())) {
6647 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USISrr_Int, RC: &X86::GR32RegClass, Op0);
6648 }
6649 return Register();
6650}
6651
6652Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i64_r(Register Op0) {
6653 if ((Subtarget->hasAVX10_2())) {
6654 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6655 }
6656 return Register();
6657}
6658
6659Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6660switch (RetVT.SimpleTy) {
6661 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i32_r(Op0);
6662 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i64_r(Op0);
6663 default: return Register();
6664}
6665}
6666
6667Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i32_r(Register Op0) {
6668 if ((Subtarget->hasAVX10_2())) {
6669 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USISrr_Int, RC: &X86::GR32RegClass, Op0);
6670 }
6671 return Register();
6672}
6673
6674Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i64_r(Register Op0) {
6675 if ((Subtarget->hasAVX10_2())) {
6676 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6677 }
6678 return Register();
6679}
6680
6681Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6682switch (RetVT.SimpleTy) {
6683 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i32_r(Op0);
6684 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i64_r(Op0);
6685 default: return Register();
6686}
6687}
6688
6689Register fastEmit_X86ISD_CVTTS2UIS_r(MVT VT, MVT RetVT, Register Op0) {
6690 switch (VT.SimpleTy) {
6691 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_r(RetVT, Op0);
6692 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_r(RetVT, Op0);
6693 default: return Register();
6694 }
6695}
6696
6697// FastEmit functions for X86ISD::CVTTS2UIS_SAE.
6698
6699Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6700 if ((Subtarget->hasAVX10_2())) {
6701 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USISrrb_Int, RC: &X86::GR32RegClass, Op0);
6702 }
6703 return Register();
6704}
6705
6706Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6707 if ((Subtarget->hasAVX10_2())) {
6708 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6709 }
6710 return Register();
6711}
6712
6713Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6714switch (RetVT.SimpleTy) {
6715 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i32_r(Op0);
6716 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i64_r(Op0);
6717 default: return Register();
6718}
6719}
6720
6721Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6722 if ((Subtarget->hasAVX10_2())) {
6723 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USISrrb_Int, RC: &X86::GR32RegClass, Op0);
6724 }
6725 return Register();
6726}
6727
6728Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6729 if ((Subtarget->hasAVX10_2())) {
6730 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6731 }
6732 return Register();
6733}
6734
6735Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6736switch (RetVT.SimpleTy) {
6737 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i32_r(Op0);
6738 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i64_r(Op0);
6739 default: return Register();
6740}
6741}
6742
6743Register fastEmit_X86ISD_CVTTS2UIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6744 switch (VT.SimpleTy) {
6745 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_r(RetVT, Op0);
6746 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_r(RetVT, Op0);
6747 default: return Register();
6748 }
6749}
6750
6751// FastEmit functions for X86ISD::CVTTS2UI_SAE.
6752
6753Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Register Op0) {
6754 if ((Subtarget->hasFP16())) {
6755 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6756 }
6757 return Register();
6758}
6759
6760Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Register Op0) {
6761 if ((Subtarget->hasFP16())) {
6762 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6763 }
6764 return Register();
6765}
6766
6767Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6768switch (RetVT.SimpleTy) {
6769 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Op0);
6770 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Op0);
6771 default: return Register();
6772}
6773}
6774
6775Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6776 if ((Subtarget->hasAVX512())) {
6777 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6778 }
6779 return Register();
6780}
6781
6782Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6783 if ((Subtarget->hasAVX512())) {
6784 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6785 }
6786 return Register();
6787}
6788
6789Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6790switch (RetVT.SimpleTy) {
6791 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Op0);
6792 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Op0);
6793 default: return Register();
6794}
6795}
6796
6797Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6798 if ((Subtarget->hasAVX512())) {
6799 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6800 }
6801 return Register();
6802}
6803
6804Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6805 if ((Subtarget->hasAVX512())) {
6806 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6807 }
6808 return Register();
6809}
6810
6811Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6812switch (RetVT.SimpleTy) {
6813 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Op0);
6814 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Op0);
6815 default: return Register();
6816}
6817}
6818
6819Register fastEmit_X86ISD_CVTTS2UI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6820 switch (VT.SimpleTy) {
6821 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(RetVT, Op0);
6822 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(RetVT, Op0);
6823 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(RetVT, Op0);
6824 default: return Register();
6825 }
6826}
6827
6828// FastEmit functions for X86ISD::CVTUI2P.
6829
6830Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
6831 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6832 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6833 }
6834 return Register();
6835}
6836
6837Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
6838 if ((Subtarget->hasVLX())) {
6839 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
6840 }
6841 return Register();
6842}
6843
6844Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
6845switch (RetVT.SimpleTy) {
6846 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
6847 case MVT::v2f64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
6848 default: return Register();
6849}
6850}
6851
6852Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
6853 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6854 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6855 }
6856 return Register();
6857}
6858
6859Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
6860 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6861 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
6862 }
6863 return Register();
6864}
6865
6866Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
6867switch (RetVT.SimpleTy) {
6868 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
6869 case MVT::v4f32: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
6870 default: return Register();
6871}
6872}
6873
6874Register fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
6875 if (RetVT.SimpleTy != MVT::v8f16)
6876 return Register();
6877 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6878 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
6879 }
6880 return Register();
6881}
6882
6883Register fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, Register Op0) {
6884 switch (VT.SimpleTy) {
6885 case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
6886 case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
6887 case MVT::v4i64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
6888 default: return Register();
6889 }
6890}
6891
6892// FastEmit functions for X86ISD::DYN_ALLOCA.
6893
6894Register fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
6895 if (RetVT.SimpleTy != MVT::isVoid)
6896 return Register();
6897 if ((!Subtarget->isTarget64BitLP64())) {
6898 return fastEmitInst_r(MachineInstOpcode: X86::DYN_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
6899 }
6900 return Register();
6901}
6902
6903Register fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
6904 if (RetVT.SimpleTy != MVT::isVoid)
6905 return Register();
6906 if ((Subtarget->is64Bit())) {
6907 return fastEmitInst_r(MachineInstOpcode: X86::DYN_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
6908 }
6909 return Register();
6910}
6911
6912Register fastEmit_X86ISD_DYN_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
6913 switch (VT.SimpleTy) {
6914 case MVT::i32: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(RetVT, Op0);
6915 case MVT::i64: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(RetVT, Op0);
6916 default: return Register();
6917 }
6918}
6919
6920// FastEmit functions for X86ISD::EH_RETURN.
6921
6922Register fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, Register Op0) {
6923 if (RetVT.SimpleTy != MVT::isVoid)
6924 return Register();
6925 return fastEmitInst_r(MachineInstOpcode: X86::EH_RETURN, RC: &X86::GR32RegClass, Op0);
6926}
6927
6928Register fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, Register Op0) {
6929 if (RetVT.SimpleTy != MVT::isVoid)
6930 return Register();
6931 return fastEmitInst_r(MachineInstOpcode: X86::EH_RETURN64, RC: &X86::GR64RegClass, Op0);
6932}
6933
6934Register fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, Register Op0) {
6935 switch (VT.SimpleTy) {
6936 case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0);
6937 case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0);
6938 default: return Register();
6939 }
6940}
6941
6942// FastEmit functions for X86ISD::FGETEXP.
6943
6944Register fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(MVT RetVT, Register Op0) {
6945 if (RetVT.SimpleTy != MVT::v8f16)
6946 return Register();
6947 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6948 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ128r, RC: &X86::VR128XRegClass, Op0);
6949 }
6950 return Register();
6951}
6952
6953Register fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(MVT RetVT, Register Op0) {
6954 if (RetVT.SimpleTy != MVT::v16f16)
6955 return Register();
6956 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6957 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ256r, RC: &X86::VR256XRegClass, Op0);
6958 }
6959 return Register();
6960}
6961
6962Register fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(MVT RetVT, Register Op0) {
6963 if (RetVT.SimpleTy != MVT::v32f16)
6964 return Register();
6965 if ((Subtarget->hasFP16())) {
6966 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZr, RC: &X86::VR512RegClass, Op0);
6967 }
6968 return Register();
6969}
6970
6971Register fastEmit_X86ISD_FGETEXP_MVT_v8bf16_r(MVT RetVT, Register Op0) {
6972 if (RetVT.SimpleTy != MVT::v8bf16)
6973 return Register();
6974 if ((Subtarget->hasAVX10_2())) {
6975 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Z128r, RC: &X86::VR128XRegClass, Op0);
6976 }
6977 return Register();
6978}
6979
6980Register fastEmit_X86ISD_FGETEXP_MVT_v16bf16_r(MVT RetVT, Register Op0) {
6981 if (RetVT.SimpleTy != MVT::v16bf16)
6982 return Register();
6983 if ((Subtarget->hasAVX10_2())) {
6984 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Z256r, RC: &X86::VR256XRegClass, Op0);
6985 }
6986 return Register();
6987}
6988
6989Register fastEmit_X86ISD_FGETEXP_MVT_v32bf16_r(MVT RetVT, Register Op0) {
6990 if (RetVT.SimpleTy != MVT::v32bf16)
6991 return Register();
6992 if ((Subtarget->hasAVX10_2())) {
6993 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Zr, RC: &X86::VR512RegClass, Op0);
6994 }
6995 return Register();
6996}
6997
6998Register fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(MVT RetVT, Register Op0) {
6999 if (RetVT.SimpleTy != MVT::v4f32)
7000 return Register();
7001 if ((Subtarget->hasVLX())) {
7002 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ128r, RC: &X86::VR128XRegClass, Op0);
7003 }
7004 return Register();
7005}
7006
7007Register fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7008 if (RetVT.SimpleTy != MVT::v8f32)
7009 return Register();
7010 if ((Subtarget->hasVLX())) {
7011 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ256r, RC: &X86::VR256XRegClass, Op0);
7012 }
7013 return Register();
7014}
7015
7016Register fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7017 if (RetVT.SimpleTy != MVT::v16f32)
7018 return Register();
7019 if ((Subtarget->hasAVX512())) {
7020 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZr, RC: &X86::VR512RegClass, Op0);
7021 }
7022 return Register();
7023}
7024
7025Register fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(MVT RetVT, Register Op0) {
7026 if (RetVT.SimpleTy != MVT::v2f64)
7027 return Register();
7028 if ((Subtarget->hasVLX())) {
7029 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ128r, RC: &X86::VR128XRegClass, Op0);
7030 }
7031 return Register();
7032}
7033
7034Register fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(MVT RetVT, Register Op0) {
7035 if (RetVT.SimpleTy != MVT::v4f64)
7036 return Register();
7037 if ((Subtarget->hasVLX())) {
7038 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ256r, RC: &X86::VR256XRegClass, Op0);
7039 }
7040 return Register();
7041}
7042
7043Register fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(MVT RetVT, Register Op0) {
7044 if (RetVT.SimpleTy != MVT::v8f64)
7045 return Register();
7046 if ((Subtarget->hasAVX512())) {
7047 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZr, RC: &X86::VR512RegClass, Op0);
7048 }
7049 return Register();
7050}
7051
7052Register fastEmit_X86ISD_FGETEXP_r(MVT VT, MVT RetVT, Register Op0) {
7053 switch (VT.SimpleTy) {
7054 case MVT::v8f16: return fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(RetVT, Op0);
7055 case MVT::v16f16: return fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(RetVT, Op0);
7056 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(RetVT, Op0);
7057 case MVT::v8bf16: return fastEmit_X86ISD_FGETEXP_MVT_v8bf16_r(RetVT, Op0);
7058 case MVT::v16bf16: return fastEmit_X86ISD_FGETEXP_MVT_v16bf16_r(RetVT, Op0);
7059 case MVT::v32bf16: return fastEmit_X86ISD_FGETEXP_MVT_v32bf16_r(RetVT, Op0);
7060 case MVT::v4f32: return fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(RetVT, Op0);
7061 case MVT::v8f32: return fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(RetVT, Op0);
7062 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(RetVT, Op0);
7063 case MVT::v2f64: return fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(RetVT, Op0);
7064 case MVT::v4f64: return fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(RetVT, Op0);
7065 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(RetVT, Op0);
7066 default: return Register();
7067 }
7068}
7069
7070// FastEmit functions for X86ISD::FGETEXP_SAE.
7071
7072Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
7073 if (RetVT.SimpleTy != MVT::v32f16)
7074 return Register();
7075 if ((Subtarget->hasFP16())) {
7076 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZrb, RC: &X86::VR512RegClass, Op0);
7077 }
7078 return Register();
7079}
7080
7081Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
7082 if (RetVT.SimpleTy != MVT::v16f32)
7083 return Register();
7084 if ((Subtarget->hasAVX512())) {
7085 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZrb, RC: &X86::VR512RegClass, Op0);
7086 }
7087 return Register();
7088}
7089
7090Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
7091 if (RetVT.SimpleTy != MVT::v8f64)
7092 return Register();
7093 if ((Subtarget->hasAVX512())) {
7094 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZrb, RC: &X86::VR512RegClass, Op0);
7095 }
7096 return Register();
7097}
7098
7099Register fastEmit_X86ISD_FGETEXP_SAE_r(MVT VT, MVT RetVT, Register Op0) {
7100 switch (VT.SimpleTy) {
7101 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(RetVT, Op0);
7102 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(RetVT, Op0);
7103 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(RetVT, Op0);
7104 default: return Register();
7105 }
7106}
7107
7108// FastEmit functions for X86ISD::FP_TO_SINT_SAT.
7109
7110Register fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7111 if (RetVT.SimpleTy != MVT::v2i64)
7112 return Register();
7113 if ((Subtarget->hasAVX10_2())) {
7114 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7115 }
7116 return Register();
7117}
7118
7119Register fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
7120 if (RetVT.SimpleTy != MVT::v4i32)
7121 return Register();
7122 if ((Subtarget->hasAVX10_2())) {
7123 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7124 }
7125 return Register();
7126}
7127
7128Register fastEmit_X86ISD_FP_TO_SINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
7129 switch (VT.SimpleTy) {
7130 case MVT::v4f32: return fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v4f32_r(RetVT, Op0);
7131 case MVT::v2f64: return fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v2f64_r(RetVT, Op0);
7132 default: return Register();
7133 }
7134}
7135
7136// FastEmit functions for X86ISD::FP_TO_UINT_SAT.
7137
7138Register fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7139 if (RetVT.SimpleTy != MVT::v2i64)
7140 return Register();
7141 if ((Subtarget->hasAVX10_2())) {
7142 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7143 }
7144 return Register();
7145}
7146
7147Register fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
7148 if (RetVT.SimpleTy != MVT::v4i32)
7149 return Register();
7150 if ((Subtarget->hasAVX10_2())) {
7151 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7152 }
7153 return Register();
7154}
7155
7156Register fastEmit_X86ISD_FP_TO_UINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
7157 switch (VT.SimpleTy) {
7158 case MVT::v4f32: return fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v4f32_r(RetVT, Op0);
7159 case MVT::v2f64: return fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v2f64_r(RetVT, Op0);
7160 default: return Register();
7161 }
7162}
7163
7164// FastEmit functions for X86ISD::FRCP.
7165
7166Register fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, Register Op0) {
7167 if (RetVT.SimpleTy != MVT::f32)
7168 return Register();
7169 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7170 return fastEmitInst_r(MachineInstOpcode: X86::RCPSSr, RC: &X86::FR32RegClass, Op0);
7171 }
7172 return Register();
7173}
7174
7175Register fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7176 if (RetVT.SimpleTy != MVT::v4f32)
7177 return Register();
7178 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7179 return fastEmitInst_r(MachineInstOpcode: X86::RCPPSr, RC: &X86::VR128RegClass, Op0);
7180 }
7181 if ((Subtarget->hasAVX())) {
7182 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSr, RC: &X86::VR128RegClass, Op0);
7183 }
7184 return Register();
7185}
7186
7187Register fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7188 if (RetVT.SimpleTy != MVT::v8f32)
7189 return Register();
7190 if ((Subtarget->hasAVX())) {
7191 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSYr, RC: &X86::VR256RegClass, Op0);
7192 }
7193 return Register();
7194}
7195
7196Register fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, Register Op0) {
7197 switch (VT.SimpleTy) {
7198 case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0);
7199 case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0);
7200 case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0);
7201 default: return Register();
7202 }
7203}
7204
7205// FastEmit functions for X86ISD::FRSQRT.
7206
7207Register fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
7208 if (RetVT.SimpleTy != MVT::f32)
7209 return Register();
7210 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7211 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTSSr, RC: &X86::FR32RegClass, Op0);
7212 }
7213 return Register();
7214}
7215
7216Register fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7217 if (RetVT.SimpleTy != MVT::v4f32)
7218 return Register();
7219 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7220 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTPSr, RC: &X86::VR128RegClass, Op0);
7221 }
7222 if ((Subtarget->hasAVX())) {
7223 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSr, RC: &X86::VR128RegClass, Op0);
7224 }
7225 return Register();
7226}
7227
7228Register fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
7229 if (RetVT.SimpleTy != MVT::v8f32)
7230 return Register();
7231 if ((Subtarget->hasAVX())) {
7232 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSYr, RC: &X86::VR256RegClass, Op0);
7233 }
7234 return Register();
7235}
7236
7237Register fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, Register Op0) {
7238 switch (VT.SimpleTy) {
7239 case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0);
7240 case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0);
7241 case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0);
7242 default: return Register();
7243 }
7244}
7245
7246// FastEmit functions for X86ISD::MMX_MOVD2W.
7247
7248Register fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, Register Op0) {
7249 if (RetVT.SimpleTy != MVT::i32)
7250 return Register();
7251 if ((Subtarget->hasMMX())) {
7252 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64grr, RC: &X86::GR32RegClass, Op0);
7253 }
7254 return Register();
7255}
7256
7257Register fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, Register Op0) {
7258 switch (VT.SimpleTy) {
7259 case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0);
7260 default: return Register();
7261 }
7262}
7263
7264// FastEmit functions for X86ISD::MMX_MOVW2D.
7265
7266Register fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, Register Op0) {
7267 if (RetVT.SimpleTy != MVT::x86mmx)
7268 return Register();
7269 if ((Subtarget->hasMMX())) {
7270 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64rr, RC: &X86::VR64RegClass, Op0);
7271 }
7272 return Register();
7273}
7274
7275Register fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, Register Op0) {
7276 switch (VT.SimpleTy) {
7277 case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0);
7278 default: return Register();
7279 }
7280}
7281
7282// FastEmit functions for X86ISD::MOVDDUP.
7283
7284Register fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, Register Op0) {
7285 if (RetVT.SimpleTy != MVT::v2f64)
7286 return Register();
7287 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7288 return fastEmitInst_r(MachineInstOpcode: X86::MOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7289 }
7290 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7291 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7292 }
7293 return Register();
7294}
7295
7296Register fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, Register Op0) {
7297 if (RetVT.SimpleTy != MVT::v4f64)
7298 return Register();
7299 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7300 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7301 }
7302 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7303 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPYrr, RC: &X86::VR256RegClass, Op0);
7304 }
7305 return Register();
7306}
7307
7308Register fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, Register Op0) {
7309 if (RetVT.SimpleTy != MVT::v8f64)
7310 return Register();
7311 if ((Subtarget->hasAVX512())) {
7312 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZrr, RC: &X86::VR512RegClass, Op0);
7313 }
7314 return Register();
7315}
7316
7317Register fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, Register Op0) {
7318 switch (VT.SimpleTy) {
7319 case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0);
7320 case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0);
7321 case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0);
7322 default: return Register();
7323 }
7324}
7325
7326// FastEmit functions for X86ISD::MOVDQ2Q.
7327
7328Register fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, Register Op0) {
7329 if (RetVT.SimpleTy != MVT::x86mmx)
7330 return Register();
7331 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
7332 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVDQ2Qrr, RC: &X86::VR64RegClass, Op0);
7333 }
7334 return Register();
7335}
7336
7337Register fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, Register Op0) {
7338 switch (VT.SimpleTy) {
7339 case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0);
7340 default: return Register();
7341 }
7342}
7343
7344// FastEmit functions for X86ISD::MOVMSK.
7345
7346Register fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, Register Op0) {
7347 if (RetVT.SimpleTy != MVT::i32)
7348 return Register();
7349 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7350 return fastEmitInst_r(MachineInstOpcode: X86::PMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
7351 }
7352 if ((Subtarget->hasAVX())) {
7353 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
7354 }
7355 return Register();
7356}
7357
7358Register fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, Register Op0) {
7359 if (RetVT.SimpleTy != MVT::i32)
7360 return Register();
7361 if ((Subtarget->hasAVX2())) {
7362 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBYrr, RC: &X86::GR32RegClass, Op0);
7363 }
7364 return Register();
7365}
7366
7367Register fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, Register Op0) {
7368 if (RetVT.SimpleTy != MVT::i32)
7369 return Register();
7370 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7371 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7372 }
7373 if ((Subtarget->hasAVX())) {
7374 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7375 }
7376 return Register();
7377}
7378
7379Register fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, Register Op0) {
7380 if (RetVT.SimpleTy != MVT::i32)
7381 return Register();
7382 if ((Subtarget->hasAVX())) {
7383 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
7384 }
7385 return Register();
7386}
7387
7388Register fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, Register Op0) {
7389 if (RetVT.SimpleTy != MVT::i32)
7390 return Register();
7391 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7392 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7393 }
7394 if ((Subtarget->hasAVX())) {
7395 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7396 }
7397 return Register();
7398}
7399
7400Register fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, Register Op0) {
7401 if (RetVT.SimpleTy != MVT::i32)
7402 return Register();
7403 if ((Subtarget->hasAVX())) {
7404 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
7405 }
7406 return Register();
7407}
7408
7409Register fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, Register Op0) {
7410 if (RetVT.SimpleTy != MVT::i32)
7411 return Register();
7412 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7413 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7414 }
7415 if ((Subtarget->hasAVX())) {
7416 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7417 }
7418 return Register();
7419}
7420
7421Register fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, Register Op0) {
7422 if (RetVT.SimpleTy != MVT::i32)
7423 return Register();
7424 if ((Subtarget->hasAVX())) {
7425 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
7426 }
7427 return Register();
7428}
7429
7430Register fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, Register Op0) {
7431 if (RetVT.SimpleTy != MVT::i32)
7432 return Register();
7433 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7434 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7435 }
7436 if ((Subtarget->hasAVX())) {
7437 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7438 }
7439 return Register();
7440}
7441
7442Register fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, Register Op0) {
7443 if (RetVT.SimpleTy != MVT::i32)
7444 return Register();
7445 if ((Subtarget->hasAVX())) {
7446 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
7447 }
7448 return Register();
7449}
7450
7451Register fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, Register Op0) {
7452 switch (VT.SimpleTy) {
7453 case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0);
7454 case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0);
7455 case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0);
7456 case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0);
7457 case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0);
7458 case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0);
7459 case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0);
7460 case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0);
7461 case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0);
7462 case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0);
7463 default: return Register();
7464 }
7465}
7466
7467// FastEmit functions for X86ISD::MOVQ2DQ.
7468
7469Register fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(MVT RetVT, Register Op0) {
7470 if (RetVT.SimpleTy != MVT::v2i64)
7471 return Register();
7472 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
7473 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2DQrr, RC: &X86::VR128RegClass, Op0);
7474 }
7475 return Register();
7476}
7477
7478Register fastEmit_X86ISD_MOVQ2DQ_r(MVT VT, MVT RetVT, Register Op0) {
7479 switch (VT.SimpleTy) {
7480 case MVT::x86mmx: return fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(RetVT, Op0);
7481 default: return Register();
7482 }
7483}
7484
7485// FastEmit functions for X86ISD::MOVSHDUP.
7486
7487Register fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, Register Op0) {
7488 if (RetVT.SimpleTy != MVT::v4i32)
7489 return Register();
7490 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7491 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7492 }
7493 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7494 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7495 }
7496 return Register();
7497}
7498
7499Register fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, Register Op0) {
7500 if (RetVT.SimpleTy != MVT::v8i32)
7501 return Register();
7502 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7503 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
7504 }
7505 return Register();
7506}
7507
7508Register fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7509 if (RetVT.SimpleTy != MVT::v4f32)
7510 return Register();
7511 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7512 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7513 }
7514 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7515 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7516 }
7517 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7518 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7519 }
7520 return Register();
7521}
7522
7523Register fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7524 if (RetVT.SimpleTy != MVT::v8f32)
7525 return Register();
7526 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7527 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7528 }
7529 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7530 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
7531 }
7532 return Register();
7533}
7534
7535Register fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7536 if (RetVT.SimpleTy != MVT::v16f32)
7537 return Register();
7538 if ((Subtarget->hasAVX512())) {
7539 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZrr, RC: &X86::VR512RegClass, Op0);
7540 }
7541 return Register();
7542}
7543
7544Register fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, Register Op0) {
7545 switch (VT.SimpleTy) {
7546 case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0);
7547 case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0);
7548 case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0);
7549 case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0);
7550 case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0);
7551 default: return Register();
7552 }
7553}
7554
7555// FastEmit functions for X86ISD::MOVSLDUP.
7556
7557Register fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, Register Op0) {
7558 if (RetVT.SimpleTy != MVT::v4i32)
7559 return Register();
7560 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7561 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7562 }
7563 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7564 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7565 }
7566 return Register();
7567}
7568
7569Register fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, Register Op0) {
7570 if (RetVT.SimpleTy != MVT::v8i32)
7571 return Register();
7572 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7573 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
7574 }
7575 return Register();
7576}
7577
7578Register fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7579 if (RetVT.SimpleTy != MVT::v4f32)
7580 return Register();
7581 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7582 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7583 }
7584 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7585 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7586 }
7587 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7588 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7589 }
7590 return Register();
7591}
7592
7593Register fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7594 if (RetVT.SimpleTy != MVT::v8f32)
7595 return Register();
7596 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7597 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7598 }
7599 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7600 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
7601 }
7602 return Register();
7603}
7604
7605Register fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7606 if (RetVT.SimpleTy != MVT::v16f32)
7607 return Register();
7608 if ((Subtarget->hasAVX512())) {
7609 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZrr, RC: &X86::VR512RegClass, Op0);
7610 }
7611 return Register();
7612}
7613
7614Register fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, Register Op0) {
7615 switch (VT.SimpleTy) {
7616 case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0);
7617 case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0);
7618 case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0);
7619 case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0);
7620 case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0);
7621 default: return Register();
7622 }
7623}
7624
7625// FastEmit functions for X86ISD::NT_BRIND.
7626
7627Register fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, Register Op0) {
7628 if (RetVT.SimpleTy != MVT::isVoid)
7629 return Register();
7630 if ((!Subtarget->is64Bit())) {
7631 return fastEmitInst_r(MachineInstOpcode: X86::JMP16r_NT, RC: &X86::GR16RegClass, Op0);
7632 }
7633 return Register();
7634}
7635
7636Register fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
7637 if (RetVT.SimpleTy != MVT::isVoid)
7638 return Register();
7639 if ((!Subtarget->is64Bit())) {
7640 return fastEmitInst_r(MachineInstOpcode: X86::JMP32r_NT, RC: &X86::GR32RegClass, Op0);
7641 }
7642 return Register();
7643}
7644
7645Register fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
7646 if (RetVT.SimpleTy != MVT::isVoid)
7647 return Register();
7648 if ((Subtarget->is64Bit())) {
7649 return fastEmitInst_r(MachineInstOpcode: X86::JMP64r_NT, RC: &X86::GR64RegClass, Op0);
7650 }
7651 return Register();
7652}
7653
7654Register fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
7655 switch (VT.SimpleTy) {
7656 case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0);
7657 case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0);
7658 case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0);
7659 default: return Register();
7660 }
7661}
7662
7663// FastEmit functions for X86ISD::NT_CALL.
7664
7665Register fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, Register Op0) {
7666 if (RetVT.SimpleTy != MVT::isVoid)
7667 return Register();
7668 if ((!Subtarget->is64Bit())) {
7669 return fastEmitInst_r(MachineInstOpcode: X86::CALL16r_NT, RC: &X86::GR16RegClass, Op0);
7670 }
7671 return Register();
7672}
7673
7674Register fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
7675 if (RetVT.SimpleTy != MVT::isVoid)
7676 return Register();
7677 if ((!Subtarget->is64Bit())) {
7678 return fastEmitInst_r(MachineInstOpcode: X86::CALL32r_NT, RC: &X86::GR32RegClass, Op0);
7679 }
7680 return Register();
7681}
7682
7683Register fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
7684 if (RetVT.SimpleTy != MVT::isVoid)
7685 return Register();
7686 if ((Subtarget->is64Bit())) {
7687 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r_NT, RC: &X86::GR64RegClass, Op0);
7688 }
7689 return Register();
7690}
7691
7692Register fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, Register Op0) {
7693 switch (VT.SimpleTy) {
7694 case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0);
7695 case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0);
7696 case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0);
7697 default: return Register();
7698 }
7699}
7700
7701// FastEmit functions for X86ISD::PHMINPOS.
7702
7703Register fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, Register Op0) {
7704 if (RetVT.SimpleTy != MVT::v8i16)
7705 return Register();
7706 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
7707 return fastEmitInst_r(MachineInstOpcode: X86::PHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
7708 }
7709 if ((Subtarget->hasAVX())) {
7710 return fastEmitInst_r(MachineInstOpcode: X86::VPHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
7711 }
7712 return Register();
7713}
7714
7715Register fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, Register Op0) {
7716 switch (VT.SimpleTy) {
7717 case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0);
7718 default: return Register();
7719 }
7720}
7721
7722// FastEmit functions for X86ISD::PROBED_ALLOCA.
7723
7724Register fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
7725 if (RetVT.SimpleTy != MVT::i32)
7726 return Register();
7727 if ((!Subtarget->isTarget64BitLP64())) {
7728 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
7729 }
7730 return Register();
7731}
7732
7733Register fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
7734 if (RetVT.SimpleTy != MVT::i64)
7735 return Register();
7736 if ((Subtarget->is64Bit())) {
7737 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
7738 }
7739 return Register();
7740}
7741
7742Register fastEmit_X86ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
7743 switch (VT.SimpleTy) {
7744 case MVT::i32: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(RetVT, Op0);
7745 case MVT::i64: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
7746 default: return Register();
7747 }
7748}
7749
7750// FastEmit functions for X86ISD::RCP14.
7751
7752Register fastEmit_X86ISD_RCP14_MVT_v8f16_r(MVT RetVT, Register Op0) {
7753 if (RetVT.SimpleTy != MVT::v8f16)
7754 return Register();
7755 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7756 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ128r, RC: &X86::VR128XRegClass, Op0);
7757 }
7758 return Register();
7759}
7760
7761Register fastEmit_X86ISD_RCP14_MVT_v16f16_r(MVT RetVT, Register Op0) {
7762 if (RetVT.SimpleTy != MVT::v16f16)
7763 return Register();
7764 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7765 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ256r, RC: &X86::VR256XRegClass, Op0);
7766 }
7767 return Register();
7768}
7769
7770Register fastEmit_X86ISD_RCP14_MVT_v32f16_r(MVT RetVT, Register Op0) {
7771 if (RetVT.SimpleTy != MVT::v32f16)
7772 return Register();
7773 if ((Subtarget->hasFP16())) {
7774 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZr, RC: &X86::VR512RegClass, Op0);
7775 }
7776 return Register();
7777}
7778
7779Register fastEmit_X86ISD_RCP14_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7780 if (RetVT.SimpleTy != MVT::v8bf16)
7781 return Register();
7782 if ((Subtarget->hasAVX10_2())) {
7783 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7784 }
7785 return Register();
7786}
7787
7788Register fastEmit_X86ISD_RCP14_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7789 if (RetVT.SimpleTy != MVT::v16bf16)
7790 return Register();
7791 if ((Subtarget->hasAVX10_2())) {
7792 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7793 }
7794 return Register();
7795}
7796
7797Register fastEmit_X86ISD_RCP14_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7798 if (RetVT.SimpleTy != MVT::v32bf16)
7799 return Register();
7800 if ((Subtarget->hasAVX10_2())) {
7801 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Zr, RC: &X86::VR512RegClass, Op0);
7802 }
7803 return Register();
7804}
7805
7806Register fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, Register Op0) {
7807 if (RetVT.SimpleTy != MVT::v4f32)
7808 return Register();
7809 if ((Subtarget->hasVLX())) {
7810 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ128r, RC: &X86::VR128XRegClass, Op0);
7811 }
7812 return Register();
7813}
7814
7815Register fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, Register Op0) {
7816 if (RetVT.SimpleTy != MVT::v8f32)
7817 return Register();
7818 if ((Subtarget->hasVLX())) {
7819 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ256r, RC: &X86::VR256XRegClass, Op0);
7820 }
7821 return Register();
7822}
7823
7824Register fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, Register Op0) {
7825 if (RetVT.SimpleTy != MVT::v16f32)
7826 return Register();
7827 if ((Subtarget->hasAVX512())) {
7828 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZr, RC: &X86::VR512RegClass, Op0);
7829 }
7830 return Register();
7831}
7832
7833Register fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, Register Op0) {
7834 if (RetVT.SimpleTy != MVT::v2f64)
7835 return Register();
7836 if ((Subtarget->hasVLX())) {
7837 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ128r, RC: &X86::VR128XRegClass, Op0);
7838 }
7839 return Register();
7840}
7841
7842Register fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, Register Op0) {
7843 if (RetVT.SimpleTy != MVT::v4f64)
7844 return Register();
7845 if ((Subtarget->hasVLX())) {
7846 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ256r, RC: &X86::VR256XRegClass, Op0);
7847 }
7848 return Register();
7849}
7850
7851Register fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, Register Op0) {
7852 if (RetVT.SimpleTy != MVT::v8f64)
7853 return Register();
7854 if ((Subtarget->hasAVX512())) {
7855 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZr, RC: &X86::VR512RegClass, Op0);
7856 }
7857 return Register();
7858}
7859
7860Register fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, Register Op0) {
7861 switch (VT.SimpleTy) {
7862 case MVT::v8f16: return fastEmit_X86ISD_RCP14_MVT_v8f16_r(RetVT, Op0);
7863 case MVT::v16f16: return fastEmit_X86ISD_RCP14_MVT_v16f16_r(RetVT, Op0);
7864 case MVT::v32f16: return fastEmit_X86ISD_RCP14_MVT_v32f16_r(RetVT, Op0);
7865 case MVT::v8bf16: return fastEmit_X86ISD_RCP14_MVT_v8bf16_r(RetVT, Op0);
7866 case MVT::v16bf16: return fastEmit_X86ISD_RCP14_MVT_v16bf16_r(RetVT, Op0);
7867 case MVT::v32bf16: return fastEmit_X86ISD_RCP14_MVT_v32bf16_r(RetVT, Op0);
7868 case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0);
7869 case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0);
7870 case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0);
7871 case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0);
7872 case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0);
7873 case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0);
7874 default: return Register();
7875 }
7876}
7877
7878// FastEmit functions for X86ISD::RSQRT14.
7879
7880Register fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(MVT RetVT, Register Op0) {
7881 if (RetVT.SimpleTy != MVT::v8f16)
7882 return Register();
7883 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7884 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
7885 }
7886 return Register();
7887}
7888
7889Register fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(MVT RetVT, Register Op0) {
7890 if (RetVT.SimpleTy != MVT::v16f16)
7891 return Register();
7892 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7893 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
7894 }
7895 return Register();
7896}
7897
7898Register fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(MVT RetVT, Register Op0) {
7899 if (RetVT.SimpleTy != MVT::v32f16)
7900 return Register();
7901 if ((Subtarget->hasFP16())) {
7902 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZr, RC: &X86::VR512RegClass, Op0);
7903 }
7904 return Register();
7905}
7906
7907Register fastEmit_X86ISD_RSQRT14_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7908 if (RetVT.SimpleTy != MVT::v8bf16)
7909 return Register();
7910 if ((Subtarget->hasAVX10_2())) {
7911 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7912 }
7913 return Register();
7914}
7915
7916Register fastEmit_X86ISD_RSQRT14_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7917 if (RetVT.SimpleTy != MVT::v16bf16)
7918 return Register();
7919 if ((Subtarget->hasAVX10_2())) {
7920 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7921 }
7922 return Register();
7923}
7924
7925Register fastEmit_X86ISD_RSQRT14_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7926 if (RetVT.SimpleTy != MVT::v32bf16)
7927 return Register();
7928 if ((Subtarget->hasAVX10_2())) {
7929 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
7930 }
7931 return Register();
7932}
7933
7934Register fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, Register Op0) {
7935 if (RetVT.SimpleTy != MVT::v4f32)
7936 return Register();
7937 if ((Subtarget->hasVLX())) {
7938 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ128r, RC: &X86::VR128XRegClass, Op0);
7939 }
7940 return Register();
7941}
7942
7943Register fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, Register Op0) {
7944 if (RetVT.SimpleTy != MVT::v8f32)
7945 return Register();
7946 if ((Subtarget->hasVLX())) {
7947 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ256r, RC: &X86::VR256XRegClass, Op0);
7948 }
7949 return Register();
7950}
7951
7952Register fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, Register Op0) {
7953 if (RetVT.SimpleTy != MVT::v16f32)
7954 return Register();
7955 if ((Subtarget->hasAVX512())) {
7956 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZr, RC: &X86::VR512RegClass, Op0);
7957 }
7958 return Register();
7959}
7960
7961Register fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, Register Op0) {
7962 if (RetVT.SimpleTy != MVT::v2f64)
7963 return Register();
7964 if ((Subtarget->hasVLX())) {
7965 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ128r, RC: &X86::VR128XRegClass, Op0);
7966 }
7967 return Register();
7968}
7969
7970Register fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, Register Op0) {
7971 if (RetVT.SimpleTy != MVT::v4f64)
7972 return Register();
7973 if ((Subtarget->hasVLX())) {
7974 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ256r, RC: &X86::VR256XRegClass, Op0);
7975 }
7976 return Register();
7977}
7978
7979Register fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, Register Op0) {
7980 if (RetVT.SimpleTy != MVT::v8f64)
7981 return Register();
7982 if ((Subtarget->hasAVX512())) {
7983 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZr, RC: &X86::VR512RegClass, Op0);
7984 }
7985 return Register();
7986}
7987
7988Register fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, Register Op0) {
7989 switch (VT.SimpleTy) {
7990 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(RetVT, Op0);
7991 case MVT::v16f16: return fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(RetVT, Op0);
7992 case MVT::v32f16: return fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(RetVT, Op0);
7993 case MVT::v8bf16: return fastEmit_X86ISD_RSQRT14_MVT_v8bf16_r(RetVT, Op0);
7994 case MVT::v16bf16: return fastEmit_X86ISD_RSQRT14_MVT_v16bf16_r(RetVT, Op0);
7995 case MVT::v32bf16: return fastEmit_X86ISD_RSQRT14_MVT_v32bf16_r(RetVT, Op0);
7996 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0);
7997 case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0);
7998 case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0);
7999 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0);
8000 case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0);
8001 case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0);
8002 default: return Register();
8003 }
8004}
8005
8006// FastEmit functions for X86ISD::SEG_ALLOCA.
8007
8008Register fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
8009 if (RetVT.SimpleTy != MVT::i32)
8010 return Register();
8011 if ((!Subtarget->isTarget64BitLP64())) {
8012 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
8013 }
8014 return Register();
8015}
8016
8017Register fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
8018 if (RetVT.SimpleTy != MVT::i64)
8019 return Register();
8020 if ((Subtarget->is64Bit())) {
8021 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
8022 }
8023 return Register();
8024}
8025
8026Register fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
8027 switch (VT.SimpleTy) {
8028 case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0);
8029 case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0);
8030 default: return Register();
8031 }
8032}
8033
8034// FastEmit functions for X86ISD::STRICT_CVTPH2PS.
8035
8036Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Register Op0) {
8037 if ((Subtarget->hasVLX())) {
8038 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8039 }
8040 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
8041 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
8042 }
8043 return Register();
8044}
8045
8046Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Register Op0) {
8047 if ((Subtarget->hasVLX())) {
8048 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
8049 }
8050 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
8051 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
8052 }
8053 return Register();
8054}
8055
8056Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(MVT RetVT, Register Op0) {
8057switch (RetVT.SimpleTy) {
8058 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
8059 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
8060 default: return Register();
8061}
8062}
8063
8064Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(MVT RetVT, Register Op0) {
8065 if (RetVT.SimpleTy != MVT::v16f32)
8066 return Register();
8067 if ((Subtarget->hasAVX512())) {
8068 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
8069 }
8070 return Register();
8071}
8072
8073Register fastEmit_X86ISD_STRICT_CVTPH2PS_r(MVT VT, MVT RetVT, Register Op0) {
8074 switch (VT.SimpleTy) {
8075 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
8076 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
8077 default: return Register();
8078 }
8079}
8080
8081// FastEmit functions for X86ISD::STRICT_CVTSI2P.
8082
8083Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
8084 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8085 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8086 }
8087 return Register();
8088}
8089
8090Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
8091 if ((Subtarget->hasVLX())) {
8092 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8093 }
8094 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8095 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
8096 }
8097 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8098 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
8099 }
8100 return Register();
8101}
8102
8103Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
8104switch (RetVT.SimpleTy) {
8105 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
8106 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
8107 default: return Register();
8108}
8109}
8110
8111Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
8112 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8113 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8114 }
8115 return Register();
8116}
8117
8118Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
8119 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8120 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8121 }
8122 return Register();
8123}
8124
8125Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
8126switch (RetVT.SimpleTy) {
8127 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
8128 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
8129 default: return Register();
8130}
8131}
8132
8133Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
8134 if (RetVT.SimpleTy != MVT::v8f16)
8135 return Register();
8136 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8137 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8138 }
8139 return Register();
8140}
8141
8142Register fastEmit_X86ISD_STRICT_CVTSI2P_r(MVT VT, MVT RetVT, Register Op0) {
8143 switch (VT.SimpleTy) {
8144 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
8145 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
8146 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
8147 default: return Register();
8148 }
8149}
8150
8151// FastEmit functions for X86ISD::STRICT_CVTTP2SI.
8152
8153Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
8154 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8155 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
8156 }
8157 return Register();
8158}
8159
8160Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
8161 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8162 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8163 }
8164 return Register();
8165}
8166
8167Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
8168 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8169 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
8170 }
8171 return Register();
8172}
8173
8174Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
8175 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8176 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8177 }
8178 return Register();
8179}
8180
8181Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
8182 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8183 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8184 }
8185 return Register();
8186}
8187
8188Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
8189 if ((Subtarget->hasFP16())) {
8190 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
8191 }
8192 return Register();
8193}
8194
8195Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
8196switch (RetVT.SimpleTy) {
8197 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
8198 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
8199 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
8200 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
8201 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
8202 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
8203 default: return Register();
8204}
8205}
8206
8207Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
8208 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
8210 }
8211 return Register();
8212}
8213
8214Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
8215 if ((Subtarget->hasFP16())) {
8216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
8217 }
8218 return Register();
8219}
8220
8221Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
8222switch (RetVT.SimpleTy) {
8223 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
8224 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
8225 default: return Register();
8226}
8227}
8228
8229Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
8230 if (RetVT.SimpleTy != MVT::v32i16)
8231 return Register();
8232 if ((Subtarget->hasFP16())) {
8233 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
8234 }
8235 return Register();
8236}
8237
8238Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
8239 if ((Subtarget->hasVLX())) {
8240 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8241 }
8242 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8243 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
8244 }
8245 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8246 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
8247 }
8248 return Register();
8249}
8250
8251Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
8252 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8253 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8254 }
8255 return Register();
8256}
8257
8258Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
8259 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8260 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8261 }
8262 return Register();
8263}
8264
8265Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
8266switch (RetVT.SimpleTy) {
8267 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
8268 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
8269 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
8270 default: return Register();
8271}
8272}
8273
8274Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
8275 if ((Subtarget->hasVLX())) {
8276 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
8277 }
8278 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8279 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
8280 }
8281 return Register();
8282}
8283
8284Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
8285 if ((Subtarget->hasDQI())) {
8286 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
8287 }
8288 return Register();
8289}
8290
8291Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
8292switch (RetVT.SimpleTy) {
8293 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
8294 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
8295 default: return Register();
8296}
8297}
8298
8299Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
8300 if (RetVT.SimpleTy != MVT::v16i32)
8301 return Register();
8302 if ((Subtarget->hasAVX512())) {
8303 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
8304 }
8305 return Register();
8306}
8307
8308Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
8309 if ((Subtarget->hasVLX())) {
8310 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8311 }
8312 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8313 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
8314 }
8315 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8316 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
8317 }
8318 return Register();
8319}
8320
8321Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
8322 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8323 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8324 }
8325 return Register();
8326}
8327
8328Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
8329switch (RetVT.SimpleTy) {
8330 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
8331 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
8332 default: return Register();
8333}
8334}
8335
8336Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
8337 if ((Subtarget->hasVLX())) {
8338 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
8339 }
8340 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8341 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
8342 }
8343 return Register();
8344}
8345
8346Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
8347 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8349 }
8350 return Register();
8351}
8352
8353Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
8354switch (RetVT.SimpleTy) {
8355 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
8356 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
8357 default: return Register();
8358}
8359}
8360
8361Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
8362 if ((Subtarget->hasAVX512())) {
8363 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
8364 }
8365 return Register();
8366}
8367
8368Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
8369 if ((Subtarget->hasDQI())) {
8370 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
8371 }
8372 return Register();
8373}
8374
8375Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
8376switch (RetVT.SimpleTy) {
8377 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
8378 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
8379 default: return Register();
8380}
8381}
8382
8383Register fastEmit_X86ISD_STRICT_CVTTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
8384 switch (VT.SimpleTy) {
8385 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
8386 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
8387 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
8388 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
8389 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
8390 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
8391 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
8392 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
8393 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
8394 default: return Register();
8395 }
8396}
8397
8398// FastEmit functions for X86ISD::STRICT_CVTTP2UI.
8399
8400Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
8401 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8402 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
8403 }
8404 return Register();
8405}
8406
8407Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
8408 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8409 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8410 }
8411 return Register();
8412}
8413
8414Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
8415 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8416 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
8417 }
8418 return Register();
8419}
8420
8421Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
8422 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8423 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8424 }
8425 return Register();
8426}
8427
8428Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
8429 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8430 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8431 }
8432 return Register();
8433}
8434
8435Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
8436 if ((Subtarget->hasFP16())) {
8437 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
8438 }
8439 return Register();
8440}
8441
8442Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
8443switch (RetVT.SimpleTy) {
8444 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
8445 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
8446 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
8447 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
8448 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
8449 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
8450 default: return Register();
8451}
8452}
8453
8454Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
8455 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8456 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
8457 }
8458 return Register();
8459}
8460
8461Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
8462 if ((Subtarget->hasFP16())) {
8463 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
8464 }
8465 return Register();
8466}
8467
8468Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
8469switch (RetVT.SimpleTy) {
8470 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
8471 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
8472 default: return Register();
8473}
8474}
8475
8476Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
8477 if (RetVT.SimpleTy != MVT::v32i16)
8478 return Register();
8479 if ((Subtarget->hasFP16())) {
8480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
8481 }
8482 return Register();
8483}
8484
8485Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
8486 if ((Subtarget->hasVLX())) {
8487 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8488 }
8489 return Register();
8490}
8491
8492Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
8493 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8495 }
8496 return Register();
8497}
8498
8499Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
8500 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8502 }
8503 return Register();
8504}
8505
8506Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
8507switch (RetVT.SimpleTy) {
8508 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
8509 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
8510 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
8511 default: return Register();
8512}
8513}
8514
8515Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
8516 if ((Subtarget->hasVLX())) {
8517 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
8518 }
8519 return Register();
8520}
8521
8522Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
8523 if ((Subtarget->hasDQI())) {
8524 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
8525 }
8526 return Register();
8527}
8528
8529Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
8530switch (RetVT.SimpleTy) {
8531 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
8532 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
8533 default: return Register();
8534}
8535}
8536
8537Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
8538 if (RetVT.SimpleTy != MVT::v16i32)
8539 return Register();
8540 if ((Subtarget->hasAVX512())) {
8541 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
8542 }
8543 return Register();
8544}
8545
8546Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
8547 if ((Subtarget->hasVLX())) {
8548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8549 }
8550 return Register();
8551}
8552
8553Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
8554 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8556 }
8557 return Register();
8558}
8559
8560Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
8561switch (RetVT.SimpleTy) {
8562 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
8563 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
8564 default: return Register();
8565}
8566}
8567
8568Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
8569 if ((Subtarget->hasVLX())) {
8570 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
8571 }
8572 return Register();
8573}
8574
8575Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
8576 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8578 }
8579 return Register();
8580}
8581
8582Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
8583switch (RetVT.SimpleTy) {
8584 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
8585 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
8586 default: return Register();
8587}
8588}
8589
8590Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
8591 if ((Subtarget->hasAVX512())) {
8592 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
8593 }
8594 return Register();
8595}
8596
8597Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
8598 if ((Subtarget->hasDQI())) {
8599 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
8600 }
8601 return Register();
8602}
8603
8604Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
8605switch (RetVT.SimpleTy) {
8606 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
8607 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
8608 default: return Register();
8609}
8610}
8611
8612Register fastEmit_X86ISD_STRICT_CVTTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
8613 switch (VT.SimpleTy) {
8614 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
8615 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
8616 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
8617 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
8618 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
8619 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
8620 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
8621 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
8622 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
8623 default: return Register();
8624 }
8625}
8626
8627// FastEmit functions for X86ISD::STRICT_CVTUI2P.
8628
8629Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
8630 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8631 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8632 }
8633 return Register();
8634}
8635
8636Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
8637 if ((Subtarget->hasVLX())) {
8638 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8639 }
8640 return Register();
8641}
8642
8643Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
8644switch (RetVT.SimpleTy) {
8645 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
8646 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
8647 default: return Register();
8648}
8649}
8650
8651Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
8652 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8653 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8654 }
8655 return Register();
8656}
8657
8658Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
8659 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8660 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8661 }
8662 return Register();
8663}
8664
8665Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
8666switch (RetVT.SimpleTy) {
8667 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
8668 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
8669 default: return Register();
8670}
8671}
8672
8673Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
8674 if (RetVT.SimpleTy != MVT::v8f16)
8675 return Register();
8676 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8677 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8678 }
8679 return Register();
8680}
8681
8682Register fastEmit_X86ISD_STRICT_CVTUI2P_r(MVT VT, MVT RetVT, Register Op0) {
8683 switch (VT.SimpleTy) {
8684 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
8685 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
8686 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
8687 default: return Register();
8688 }
8689}
8690
8691// FastEmit functions for X86ISD::STRICT_VFPEXT.
8692
8693Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Register Op0) {
8694 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8695 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
8696 }
8697 return Register();
8698}
8699
8700Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Register Op0) {
8701 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8702 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8703 }
8704 return Register();
8705}
8706
8707Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Register Op0) {
8708 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8709 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
8710 }
8711 return Register();
8712}
8713
8714Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(MVT RetVT, Register Op0) {
8715switch (RetVT.SimpleTy) {
8716 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
8717 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
8718 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
8719 default: return Register();
8720}
8721}
8722
8723Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(MVT RetVT, Register Op0) {
8724 if (RetVT.SimpleTy != MVT::v2f64)
8725 return Register();
8726 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8727 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8728 }
8729 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8730 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
8731 }
8732 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8733 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
8734 }
8735 return Register();
8736}
8737
8738Register fastEmit_X86ISD_STRICT_VFPEXT_r(MVT VT, MVT RetVT, Register Op0) {
8739 switch (VT.SimpleTy) {
8740 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(RetVT, Op0);
8741 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(RetVT, Op0);
8742 default: return Register();
8743 }
8744}
8745
8746// FastEmit functions for X86ISD::STRICT_VFPROUND.
8747
8748Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
8749 if (RetVT.SimpleTy != MVT::v8f16)
8750 return Register();
8751 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8752 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
8753 }
8754 return Register();
8755}
8756
8757Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(MVT RetVT, Register Op0) {
8758 if (RetVT.SimpleTy != MVT::v8f16)
8759 return Register();
8760 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8761 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
8762 }
8763 return Register();
8764}
8765
8766Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(MVT RetVT, Register Op0) {
8767 if (RetVT.SimpleTy != MVT::v16f16)
8768 return Register();
8769 if ((Subtarget->hasFP16())) {
8770 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
8771 }
8772 return Register();
8773}
8774
8775Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Register Op0) {
8776 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8777 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8778 }
8779 return Register();
8780}
8781
8782Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Register Op0) {
8783 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8784 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8785 }
8786 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8787 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8788 }
8789 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8790 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8791 }
8792 return Register();
8793}
8794
8795Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
8796switch (RetVT.SimpleTy) {
8797 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
8798 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
8799 default: return Register();
8800}
8801}
8802
8803Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Register Op0) {
8804 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8805 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8806 }
8807 return Register();
8808}
8809
8810Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Register Op0) {
8811 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8812 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
8813 }
8814 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8815 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
8816 }
8817 return Register();
8818}
8819
8820Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(MVT RetVT, Register Op0) {
8821switch (RetVT.SimpleTy) {
8822 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
8823 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
8824 default: return Register();
8825}
8826}
8827
8828Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Register Op0) {
8829 if ((Subtarget->hasFP16())) {
8830 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
8831 }
8832 return Register();
8833}
8834
8835Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Register Op0) {
8836 if ((Subtarget->hasAVX512())) {
8837 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
8838 }
8839 return Register();
8840}
8841
8842Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(MVT RetVT, Register Op0) {
8843switch (RetVT.SimpleTy) {
8844 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
8845 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
8846 default: return Register();
8847}
8848}
8849
8850Register fastEmit_X86ISD_STRICT_VFPROUND_r(MVT VT, MVT RetVT, Register Op0) {
8851 switch (VT.SimpleTy) {
8852 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(RetVT, Op0);
8853 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(RetVT, Op0);
8854 case MVT::v16f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(RetVT, Op0);
8855 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(RetVT, Op0);
8856 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(RetVT, Op0);
8857 case MVT::v8f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(RetVT, Op0);
8858 default: return Register();
8859 }
8860}
8861
8862// FastEmit functions for X86ISD::VBROADCAST.
8863
8864Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Register Op0) {
8865 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8866 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ128rr, RC: &X86::VR128XRegClass, Op0);
8867 }
8868 return Register();
8869}
8870
8871Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Register Op0) {
8872 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8873 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ256rr, RC: &X86::VR256XRegClass, Op0);
8874 }
8875 return Register();
8876}
8877
8878Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Register Op0) {
8879 if ((Subtarget->hasAVX512())) {
8880 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZrr, RC: &X86::VR512RegClass, Op0);
8881 }
8882 return Register();
8883}
8884
8885Register fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, Register Op0) {
8886switch (RetVT.SimpleTy) {
8887 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0);
8888 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0);
8889 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0);
8890 default: return Register();
8891}
8892}
8893
8894Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Register Op0) {
8895 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8896 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ128rr, RC: &X86::VR128XRegClass, Op0);
8897 }
8898 return Register();
8899}
8900
8901Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Register Op0) {
8902 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8903 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ256rr, RC: &X86::VR256XRegClass, Op0);
8904 }
8905 return Register();
8906}
8907
8908Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Register Op0) {
8909 if ((Subtarget->hasAVX512())) {
8910 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZrr, RC: &X86::VR512RegClass, Op0);
8911 }
8912 return Register();
8913}
8914
8915Register fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, Register Op0) {
8916switch (RetVT.SimpleTy) {
8917 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0);
8918 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0);
8919 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0);
8920 default: return Register();
8921}
8922}
8923
8924Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Register Op0) {
8925 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8926 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ128rr, RC: &X86::VR128XRegClass, Op0);
8927 }
8928 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8929 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBrr, RC: &X86::VR128RegClass, Op0);
8930 }
8931 return Register();
8932}
8933
8934Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Register Op0) {
8935 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8936 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ256rr, RC: &X86::VR256XRegClass, Op0);
8937 }
8938 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8939 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBYrr, RC: &X86::VR256RegClass, Op0);
8940 }
8941 return Register();
8942}
8943
8944Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Register Op0) {
8945 if ((Subtarget->hasBWI())) {
8946 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZrr, RC: &X86::VR512RegClass, Op0);
8947 }
8948 return Register();
8949}
8950
8951Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
8952switch (RetVT.SimpleTy) {
8953 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0);
8954 case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0);
8955 case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0);
8956 default: return Register();
8957}
8958}
8959
8960Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Register Op0) {
8961 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8962 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
8963 }
8964 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8965 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
8966 }
8967 return Register();
8968}
8969
8970Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Register Op0) {
8971 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8972 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
8973 }
8974 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8975 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
8976 }
8977 return Register();
8978}
8979
8980Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Register Op0) {
8981 if ((Subtarget->hasBWI())) {
8982 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
8983 }
8984 return Register();
8985}
8986
8987Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
8988switch (RetVT.SimpleTy) {
8989 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0);
8990 case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0);
8991 case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0);
8992 default: return Register();
8993}
8994}
8995
8996Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Register Op0) {
8997 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8998 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ128rr, RC: &X86::VR128XRegClass, Op0);
8999 }
9000 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9001 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrr, RC: &X86::VR128RegClass, Op0);
9002 }
9003 return Register();
9004}
9005
9006Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Register Op0) {
9007 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9008 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ256rr, RC: &X86::VR256XRegClass, Op0);
9009 }
9010 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9011 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDYrr, RC: &X86::VR256RegClass, Op0);
9012 }
9013 return Register();
9014}
9015
9016Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Register Op0) {
9017 if ((Subtarget->hasAVX512())) {
9018 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZrr, RC: &X86::VR512RegClass, Op0);
9019 }
9020 return Register();
9021}
9022
9023Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
9024switch (RetVT.SimpleTy) {
9025 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0);
9026 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0);
9027 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0);
9028 default: return Register();
9029}
9030}
9031
9032Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Register Op0) {
9033 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9034 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ128rr, RC: &X86::VR128XRegClass, Op0);
9035 }
9036 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9037 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrr, RC: &X86::VR128RegClass, Op0);
9038 }
9039 return Register();
9040}
9041
9042Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Register Op0) {
9043 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9044 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ256rr, RC: &X86::VR256XRegClass, Op0);
9045 }
9046 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9047 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQYrr, RC: &X86::VR256RegClass, Op0);
9048 }
9049 return Register();
9050}
9051
9052Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Register Op0) {
9053 if ((Subtarget->hasAVX512())) {
9054 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZrr, RC: &X86::VR512RegClass, Op0);
9055 }
9056 return Register();
9057}
9058
9059Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
9060switch (RetVT.SimpleTy) {
9061 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0);
9062 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0);
9063 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0);
9064 default: return Register();
9065}
9066}
9067
9068Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Register Op0) {
9069 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9070 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
9071 }
9072 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9073 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
9074 }
9075 return Register();
9076}
9077
9078Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Register Op0) {
9079 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9080 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
9081 }
9082 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9083 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
9084 }
9085 return Register();
9086}
9087
9088Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Register Op0) {
9089 if ((Subtarget->hasBWI())) {
9090 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
9091 }
9092 return Register();
9093}
9094
9095Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
9096switch (RetVT.SimpleTy) {
9097 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Op0);
9098 case MVT::v16f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Op0);
9099 case MVT::v32f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Op0);
9100 default: return Register();
9101}
9102}
9103
9104Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Register Op0) {
9105 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9106 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
9107 }
9108 return Register();
9109}
9110
9111Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Register Op0) {
9112 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9113 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
9114 }
9115 return Register();
9116}
9117
9118Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Register Op0) {
9119 if ((Subtarget->hasBF16())) {
9120 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
9121 }
9122 return Register();
9123}
9124
9125Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
9126switch (RetVT.SimpleTy) {
9127 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Op0);
9128 case MVT::v16bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Op0);
9129 case MVT::v32bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Op0);
9130 default: return Register();
9131}
9132}
9133
9134Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Register Op0) {
9135 if ((Subtarget->hasVLX())) {
9136 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ128rr, RC: &X86::VR128XRegClass, Op0);
9137 }
9138 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9139 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSrr, RC: &X86::VR128RegClass, Op0);
9140 }
9141 return Register();
9142}
9143
9144Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Register Op0) {
9145 if ((Subtarget->hasVLX())) {
9146 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ256rr, RC: &X86::VR256XRegClass, Op0);
9147 }
9148 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9149 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSYrr, RC: &X86::VR256RegClass, Op0);
9150 }
9151 return Register();
9152}
9153
9154Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Register Op0) {
9155 if ((Subtarget->hasAVX512())) {
9156 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZrr, RC: &X86::VR512RegClass, Op0);
9157 }
9158 return Register();
9159}
9160
9161Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
9162switch (RetVT.SimpleTy) {
9163 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0);
9164 case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0);
9165 case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0);
9166 default: return Register();
9167}
9168}
9169
9170Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Register Op0) {
9171 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9172 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
9173 }
9174 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9175 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
9176 }
9177 return Register();
9178}
9179
9180Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Register Op0) {
9181 if ((Subtarget->hasVLX())) {
9182 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZ256rr, RC: &X86::VR256XRegClass, Op0);
9183 }
9184 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9185 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDYrr, RC: &X86::VR256RegClass, Op0);
9186 }
9187 return Register();
9188}
9189
9190Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Register Op0) {
9191 if ((Subtarget->hasAVX512())) {
9192 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZrr, RC: &X86::VR512RegClass, Op0);
9193 }
9194 return Register();
9195}
9196
9197Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
9198switch (RetVT.SimpleTy) {
9199 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0);
9200 case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0);
9201 case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0);
9202 default: return Register();
9203}
9204}
9205
9206Register fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, Register Op0) {
9207 switch (VT.SimpleTy) {
9208 case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0);
9209 case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0);
9210 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0);
9211 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0);
9212 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0);
9213 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0);
9214 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(RetVT, Op0);
9215 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(RetVT, Op0);
9216 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0);
9217 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0);
9218 default: return Register();
9219 }
9220}
9221
9222// FastEmit functions for X86ISD::VBROADCASTM.
9223
9224Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Register Op0) {
9225 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9226 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ128rr, RC: &X86::VR128XRegClass, Op0);
9227 }
9228 return Register();
9229}
9230
9231Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Register Op0) {
9232 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9233 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ256rr, RC: &X86::VR256XRegClass, Op0);
9234 }
9235 return Register();
9236}
9237
9238Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Register Op0) {
9239 if ((Subtarget->hasCDI())) {
9240 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZrr, RC: &X86::VR512RegClass, Op0);
9241 }
9242 return Register();
9243}
9244
9245Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, Register Op0) {
9246switch (RetVT.SimpleTy) {
9247 case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0);
9248 case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0);
9249 case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0);
9250 default: return Register();
9251}
9252}
9253
9254Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Register Op0) {
9255 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9256 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ128rr, RC: &X86::VR128XRegClass, Op0);
9257 }
9258 return Register();
9259}
9260
9261Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Register Op0) {
9262 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9263 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ256rr, RC: &X86::VR256XRegClass, Op0);
9264 }
9265 return Register();
9266}
9267
9268Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Register Op0) {
9269 if ((Subtarget->hasCDI())) {
9270 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZrr, RC: &X86::VR512RegClass, Op0);
9271 }
9272 return Register();
9273}
9274
9275Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, Register Op0) {
9276switch (RetVT.SimpleTy) {
9277 case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0);
9278 case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0);
9279 case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0);
9280 default: return Register();
9281}
9282}
9283
9284Register fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, Register Op0) {
9285 switch (VT.SimpleTy) {
9286 case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0);
9287 case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0);
9288 default: return Register();
9289 }
9290}
9291
9292// FastEmit functions for X86ISD::VCVTHF82PH.
9293
9294Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v8f16_r(Register Op0) {
9295 if ((Subtarget->hasAVX10_2())) {
9296 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZ128rr, RC: &X86::VR128XRegClass, Op0);
9297 }
9298 return Register();
9299}
9300
9301Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v16f16_r(Register Op0) {
9302 if ((Subtarget->hasAVX10_2())) {
9303 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZ256rr, RC: &X86::VR256XRegClass, Op0);
9304 }
9305 return Register();
9306}
9307
9308Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_r(MVT RetVT, Register Op0) {
9309switch (RetVT.SimpleTy) {
9310 case MVT::v8f16: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v8f16_r(Op0);
9311 case MVT::v16f16: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v16f16_r(Op0);
9312 default: return Register();
9313}
9314}
9315
9316Register fastEmit_X86ISD_VCVTHF82PH_MVT_v32i8_r(MVT RetVT, Register Op0) {
9317 if (RetVT.SimpleTy != MVT::v32f16)
9318 return Register();
9319 if ((Subtarget->hasAVX10_2())) {
9320 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZrr, RC: &X86::VR512RegClass, Op0);
9321 }
9322 return Register();
9323}
9324
9325Register fastEmit_X86ISD_VCVTHF82PH_r(MVT VT, MVT RetVT, Register Op0) {
9326 switch (VT.SimpleTy) {
9327 case MVT::v16i8: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_r(RetVT, Op0);
9328 case MVT::v32i8: return fastEmit_X86ISD_VCVTHF82PH_MVT_v32i8_r(RetVT, Op0);
9329 default: return Register();
9330 }
9331}
9332
9333// FastEmit functions for X86ISD::VCVTPH2BF8.
9334
9335Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v8f16_r(MVT RetVT, Register Op0) {
9336 if (RetVT.SimpleTy != MVT::v16i8)
9337 return Register();
9338 if ((Subtarget->hasAVX10_2())) {
9339 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9340 }
9341 return Register();
9342}
9343
9344Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v16f16_r(MVT RetVT, Register Op0) {
9345 if (RetVT.SimpleTy != MVT::v16i8)
9346 return Register();
9347 if ((Subtarget->hasAVX10_2())) {
9348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9349 }
9350 return Register();
9351}
9352
9353Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v32f16_r(MVT RetVT, Register Op0) {
9354 if (RetVT.SimpleTy != MVT::v32i8)
9355 return Register();
9356 if ((Subtarget->hasAVX10_2())) {
9357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Zrr, RC: &X86::VR256XRegClass, Op0);
9358 }
9359 return Register();
9360}
9361
9362Register fastEmit_X86ISD_VCVTPH2BF8_r(MVT VT, MVT RetVT, Register Op0) {
9363 switch (VT.SimpleTy) {
9364 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v8f16_r(RetVT, Op0);
9365 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v16f16_r(RetVT, Op0);
9366 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v32f16_r(RetVT, Op0);
9367 default: return Register();
9368 }
9369}
9370
9371// FastEmit functions for X86ISD::VCVTPH2BF8S.
9372
9373Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v8f16_r(MVT RetVT, Register Op0) {
9374 if (RetVT.SimpleTy != MVT::v16i8)
9375 return Register();
9376 if ((Subtarget->hasAVX10_2())) {
9377 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9378 }
9379 return Register();
9380}
9381
9382Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v16f16_r(MVT RetVT, Register Op0) {
9383 if (RetVT.SimpleTy != MVT::v16i8)
9384 return Register();
9385 if ((Subtarget->hasAVX10_2())) {
9386 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9387 }
9388 return Register();
9389}
9390
9391Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v32f16_r(MVT RetVT, Register Op0) {
9392 if (RetVT.SimpleTy != MVT::v32i8)
9393 return Register();
9394 if ((Subtarget->hasAVX10_2())) {
9395 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZrr, RC: &X86::VR256XRegClass, Op0);
9396 }
9397 return Register();
9398}
9399
9400Register fastEmit_X86ISD_VCVTPH2BF8S_r(MVT VT, MVT RetVT, Register Op0) {
9401 switch (VT.SimpleTy) {
9402 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v8f16_r(RetVT, Op0);
9403 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v16f16_r(RetVT, Op0);
9404 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v32f16_r(RetVT, Op0);
9405 default: return Register();
9406 }
9407}
9408
9409// FastEmit functions for X86ISD::VCVTPH2HF8.
9410
9411Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v8f16_r(MVT RetVT, Register Op0) {
9412 if (RetVT.SimpleTy != MVT::v16i8)
9413 return Register();
9414 if ((Subtarget->hasAVX10_2())) {
9415 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9416 }
9417 return Register();
9418}
9419
9420Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v16f16_r(MVT RetVT, Register Op0) {
9421 if (RetVT.SimpleTy != MVT::v16i8)
9422 return Register();
9423 if ((Subtarget->hasAVX10_2())) {
9424 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9425 }
9426 return Register();
9427}
9428
9429Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v32f16_r(MVT RetVT, Register Op0) {
9430 if (RetVT.SimpleTy != MVT::v32i8)
9431 return Register();
9432 if ((Subtarget->hasAVX10_2())) {
9433 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Zrr, RC: &X86::VR256XRegClass, Op0);
9434 }
9435 return Register();
9436}
9437
9438Register fastEmit_X86ISD_VCVTPH2HF8_r(MVT VT, MVT RetVT, Register Op0) {
9439 switch (VT.SimpleTy) {
9440 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v8f16_r(RetVT, Op0);
9441 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v16f16_r(RetVT, Op0);
9442 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v32f16_r(RetVT, Op0);
9443 default: return Register();
9444 }
9445}
9446
9447// FastEmit functions for X86ISD::VCVTPH2HF8S.
9448
9449Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v8f16_r(MVT RetVT, Register Op0) {
9450 if (RetVT.SimpleTy != MVT::v16i8)
9451 return Register();
9452 if ((Subtarget->hasAVX10_2())) {
9453 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9454 }
9455 return Register();
9456}
9457
9458Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v16f16_r(MVT RetVT, Register Op0) {
9459 if (RetVT.SimpleTy != MVT::v16i8)
9460 return Register();
9461 if ((Subtarget->hasAVX10_2())) {
9462 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9463 }
9464 return Register();
9465}
9466
9467Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v32f16_r(MVT RetVT, Register Op0) {
9468 if (RetVT.SimpleTy != MVT::v32i8)
9469 return Register();
9470 if ((Subtarget->hasAVX10_2())) {
9471 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZrr, RC: &X86::VR256XRegClass, Op0);
9472 }
9473 return Register();
9474}
9475
9476Register fastEmit_X86ISD_VCVTPH2HF8S_r(MVT VT, MVT RetVT, Register Op0) {
9477 switch (VT.SimpleTy) {
9478 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v8f16_r(RetVT, Op0);
9479 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v16f16_r(RetVT, Op0);
9480 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v32f16_r(RetVT, Op0);
9481 default: return Register();
9482 }
9483}
9484
9485// FastEmit functions for X86ISD::VFPEXT.
9486
9487Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Register Op0) {
9488 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9489 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
9490 }
9491 return Register();
9492}
9493
9494Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Register Op0) {
9495 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9496 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
9497 }
9498 return Register();
9499}
9500
9501Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Register Op0) {
9502 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9503 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
9504 }
9505 return Register();
9506}
9507
9508Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(MVT RetVT, Register Op0) {
9509switch (RetVT.SimpleTy) {
9510 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
9511 case MVT::v2f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
9512 case MVT::v4f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
9513 default: return Register();
9514}
9515}
9516
9517Register fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, Register Op0) {
9518 if (RetVT.SimpleTy != MVT::v2f64)
9519 return Register();
9520 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9521 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
9522 }
9523 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9524 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
9525 }
9526 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9527 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
9528 }
9529 return Register();
9530}
9531
9532Register fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, Register Op0) {
9533 switch (VT.SimpleTy) {
9534 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(RetVT, Op0);
9535 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0);
9536 default: return Register();
9537 }
9538}
9539
9540// FastEmit functions for X86ISD::VFPEXT_SAE.
9541
9542Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
9543 if (RetVT.SimpleTy != MVT::v8f64)
9544 return Register();
9545 if ((Subtarget->hasFP16())) {
9546 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrrb, RC: &X86::VR512RegClass, Op0);
9547 }
9548 return Register();
9549}
9550
9551Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
9552 if (RetVT.SimpleTy != MVT::v16f32)
9553 return Register();
9554 if ((Subtarget->hasFP16())) {
9555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrrb, RC: &X86::VR512RegClass, Op0);
9556 }
9557 return Register();
9558}
9559
9560Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
9561 if (RetVT.SimpleTy != MVT::v8f64)
9562 return Register();
9563 if ((Subtarget->hasAVX512())) {
9564 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrrb, RC: &X86::VR512RegClass, Op0);
9565 }
9566 return Register();
9567}
9568
9569Register fastEmit_X86ISD_VFPEXT_SAE_r(MVT VT, MVT RetVT, Register Op0) {
9570 switch (VT.SimpleTy) {
9571 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(RetVT, Op0);
9572 case MVT::v16f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(RetVT, Op0);
9573 case MVT::v8f32: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(RetVT, Op0);
9574 default: return Register();
9575 }
9576}
9577
9578// FastEmit functions for X86ISD::VFPROUND.
9579
9580Register fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
9581 if (RetVT.SimpleTy != MVT::v8f16)
9582 return Register();
9583 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9584 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
9585 }
9586 return Register();
9587}
9588
9589Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Register Op0) {
9590 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9591 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
9592 }
9593 return Register();
9594}
9595
9596Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Register Op0) {
9597 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9598 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
9599 }
9600 if ((Subtarget->hasAVXNECONVERT())) {
9601 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Yrr, RC: &X86::VR128RegClass, Op0);
9602 }
9603 return Register();
9604}
9605
9606Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(MVT RetVT, Register Op0) {
9607switch (RetVT.SimpleTy) {
9608 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Op0);
9609 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Op0);
9610 default: return Register();
9611}
9612}
9613
9614Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Register Op0) {
9615 if ((Subtarget->hasFP16())) {
9616 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
9617 }
9618 return Register();
9619}
9620
9621Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Register Op0) {
9622 if ((Subtarget->hasBF16())) {
9623 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
9624 }
9625 return Register();
9626}
9627
9628Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(MVT RetVT, Register Op0) {
9629switch (RetVT.SimpleTy) {
9630 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Op0);
9631 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Op0);
9632 default: return Register();
9633}
9634}
9635
9636Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Register Op0) {
9637 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9638 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
9639 }
9640 return Register();
9641}
9642
9643Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Register Op0) {
9644 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9645 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
9646 }
9647 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9648 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
9649 }
9650 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9651 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
9652 }
9653 return Register();
9654}
9655
9656Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
9657switch (RetVT.SimpleTy) {
9658 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
9659 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
9660 default: return Register();
9661}
9662}
9663
9664Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Register Op0) {
9665 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9666 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
9667 }
9668 return Register();
9669}
9670
9671Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Register Op0) {
9672 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9673 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
9674 }
9675 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9676 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
9677 }
9678 return Register();
9679}
9680
9681Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(MVT RetVT, Register Op0) {
9682switch (RetVT.SimpleTy) {
9683 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
9684 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
9685 default: return Register();
9686}
9687}
9688
9689Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Register Op0) {
9690 if ((Subtarget->hasFP16())) {
9691 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
9692 }
9693 return Register();
9694}
9695
9696Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Register Op0) {
9697 if ((Subtarget->hasAVX512())) {
9698 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
9699 }
9700 return Register();
9701}
9702
9703Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(MVT RetVT, Register Op0) {
9704switch (RetVT.SimpleTy) {
9705 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
9706 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
9707 default: return Register();
9708}
9709}
9710
9711Register fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, Register Op0) {
9712 switch (VT.SimpleTy) {
9713 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(RetVT, Op0);
9714 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(RetVT, Op0);
9715 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(RetVT, Op0);
9716 case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0);
9717 case MVT::v4f64: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(RetVT, Op0);
9718 case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(RetVT, Op0);
9719 default: return Register();
9720 }
9721}
9722
9723// FastEmit functions for X86ISD::VTRUNC.
9724
9725Register fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, Register Op0) {
9726 if (RetVT.SimpleTy != MVT::v16i8)
9727 return Register();
9728 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9729 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ128rr, RC: &X86::VR128XRegClass, Op0);
9730 }
9731 return Register();
9732}
9733
9734Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Register Op0) {
9735 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9736 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ128rr, RC: &X86::VR128XRegClass, Op0);
9737 }
9738 return Register();
9739}
9740
9741Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Register Op0) {
9742 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9743 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ128rr, RC: &X86::VR128XRegClass, Op0);
9744 }
9745 return Register();
9746}
9747
9748Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, Register Op0) {
9749switch (RetVT.SimpleTy) {
9750 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0);
9751 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0);
9752 default: return Register();
9753}
9754}
9755
9756Register fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, Register Op0) {
9757 if (RetVT.SimpleTy != MVT::v16i8)
9758 return Register();
9759 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9760 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ256rr, RC: &X86::VR128XRegClass, Op0);
9761 }
9762 return Register();
9763}
9764
9765Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Register Op0) {
9766 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9767 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ128rr, RC: &X86::VR128XRegClass, Op0);
9768 }
9769 return Register();
9770}
9771
9772Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Register Op0) {
9773 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9774 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ128rr, RC: &X86::VR128XRegClass, Op0);
9775 }
9776 return Register();
9777}
9778
9779Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Register Op0) {
9780 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9781 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ128rr, RC: &X86::VR128XRegClass, Op0);
9782 }
9783 return Register();
9784}
9785
9786Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, Register Op0) {
9787switch (RetVT.SimpleTy) {
9788 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0);
9789 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0);
9790 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0);
9791 default: return Register();
9792}
9793}
9794
9795Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Register Op0) {
9796 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9797 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ256rr, RC: &X86::VR128XRegClass, Op0);
9798 }
9799 return Register();
9800}
9801
9802Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Register Op0) {
9803 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9804 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ256rr, RC: &X86::VR128XRegClass, Op0);
9805 }
9806 return Register();
9807}
9808
9809Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, Register Op0) {
9810switch (RetVT.SimpleTy) {
9811 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0);
9812 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0);
9813 default: return Register();
9814}
9815}
9816
9817Register fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, Register Op0) {
9818 if (RetVT.SimpleTy != MVT::v16i8)
9819 return Register();
9820 if ((Subtarget->hasAVX512())) {
9821 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZrr, RC: &X86::VR128XRegClass, Op0);
9822 }
9823 return Register();
9824}
9825
9826Register fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
9827 switch (VT.SimpleTy) {
9828 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0);
9829 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0);
9830 case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0);
9831 case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0);
9832 case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0);
9833 case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0);
9834 default: return Register();
9835 }
9836}
9837
9838// FastEmit functions for X86ISD::VTRUNCS.
9839
9840Register fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, Register Op0) {
9841 if (RetVT.SimpleTy != MVT::v16i8)
9842 return Register();
9843 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9844 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
9845 }
9846 return Register();
9847}
9848
9849Register fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, Register Op0) {
9850 if (RetVT.SimpleTy != MVT::v16i8)
9851 return Register();
9852 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9853 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
9854 }
9855 return Register();
9856}
9857
9858Register fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, Register Op0) {
9859 if (RetVT.SimpleTy != MVT::v32i8)
9860 return Register();
9861 if ((Subtarget->hasBWI())) {
9862 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZrr, RC: &X86::VR256XRegClass, Op0);
9863 }
9864 return Register();
9865}
9866
9867Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Register Op0) {
9868 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9869 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
9870 }
9871 return Register();
9872}
9873
9874Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Register Op0) {
9875 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9876 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
9877 }
9878 return Register();
9879}
9880
9881Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, Register Op0) {
9882switch (RetVT.SimpleTy) {
9883 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0);
9884 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0);
9885 default: return Register();
9886}
9887}
9888
9889Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Register Op0) {
9890 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9891 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
9892 }
9893 return Register();
9894}
9895
9896Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Register Op0) {
9897 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9898 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
9899 }
9900 return Register();
9901}
9902
9903Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, Register Op0) {
9904switch (RetVT.SimpleTy) {
9905 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0);
9906 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0);
9907 default: return Register();
9908}
9909}
9910
9911Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Register Op0) {
9912 if ((Subtarget->hasAVX512())) {
9913 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZrr, RC: &X86::VR128XRegClass, Op0);
9914 }
9915 return Register();
9916}
9917
9918Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Register Op0) {
9919 if ((Subtarget->hasAVX512())) {
9920 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZrr, RC: &X86::VR256XRegClass, Op0);
9921 }
9922 return Register();
9923}
9924
9925Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, Register Op0) {
9926switch (RetVT.SimpleTy) {
9927 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0);
9928 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0);
9929 default: return Register();
9930}
9931}
9932
9933Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Register Op0) {
9934 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9935 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
9936 }
9937 return Register();
9938}
9939
9940Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Register Op0) {
9941 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9942 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
9943 }
9944 return Register();
9945}
9946
9947Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Register Op0) {
9948 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9949 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
9950 }
9951 return Register();
9952}
9953
9954Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, Register Op0) {
9955switch (RetVT.SimpleTy) {
9956 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0);
9957 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0);
9958 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0);
9959 default: return Register();
9960}
9961}
9962
9963Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Register Op0) {
9964 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9965 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
9966 }
9967 return Register();
9968}
9969
9970Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Register Op0) {
9971 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9972 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
9973 }
9974 return Register();
9975}
9976
9977Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Register Op0) {
9978 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9979 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
9980 }
9981 return Register();
9982}
9983
9984Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, Register Op0) {
9985switch (RetVT.SimpleTy) {
9986 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0);
9987 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0);
9988 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0);
9989 default: return Register();
9990}
9991}
9992
9993Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Register Op0) {
9994 if ((Subtarget->hasAVX512())) {
9995 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZrr, RC: &X86::VR128XRegClass, Op0);
9996 }
9997 return Register();
9998}
9999
10000Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Register Op0) {
10001 if ((Subtarget->hasAVX512())) {
10002 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZrr, RC: &X86::VR128XRegClass, Op0);
10003 }
10004 return Register();
10005}
10006
10007Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Register Op0) {
10008 if ((Subtarget->hasAVX512())) {
10009 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZrr, RC: &X86::VR256XRegClass, Op0);
10010 }
10011 return Register();
10012}
10013
10014Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, Register Op0) {
10015switch (RetVT.SimpleTy) {
10016 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0);
10017 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0);
10018 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0);
10019 default: return Register();
10020}
10021}
10022
10023Register fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, Register Op0) {
10024 switch (VT.SimpleTy) {
10025 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0);
10026 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0);
10027 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0);
10028 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0);
10029 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0);
10030 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0);
10031 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0);
10032 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0);
10033 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0);
10034 default: return Register();
10035 }
10036}
10037
10038// FastEmit functions for X86ISD::VTRUNCUS.
10039
10040Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, Register Op0) {
10041 if (RetVT.SimpleTy != MVT::v16i8)
10042 return Register();
10043 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10044 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
10045 }
10046 return Register();
10047}
10048
10049Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, Register Op0) {
10050 if (RetVT.SimpleTy != MVT::v16i8)
10051 return Register();
10052 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10053 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
10054 }
10055 return Register();
10056}
10057
10058Register fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, Register Op0) {
10059 if (RetVT.SimpleTy != MVT::v32i8)
10060 return Register();
10061 if ((Subtarget->hasBWI())) {
10062 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZrr, RC: &X86::VR256XRegClass, Op0);
10063 }
10064 return Register();
10065}
10066
10067Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Register Op0) {
10068 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10069 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
10070 }
10071 return Register();
10072}
10073
10074Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Register Op0) {
10075 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10076 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
10077 }
10078 return Register();
10079}
10080
10081Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, Register Op0) {
10082switch (RetVT.SimpleTy) {
10083 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0);
10084 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0);
10085 default: return Register();
10086}
10087}
10088
10089Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Register Op0) {
10090 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10091 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
10092 }
10093 return Register();
10094}
10095
10096Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Register Op0) {
10097 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10098 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
10099 }
10100 return Register();
10101}
10102
10103Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, Register Op0) {
10104switch (RetVT.SimpleTy) {
10105 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0);
10106 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0);
10107 default: return Register();
10108}
10109}
10110
10111Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Register Op0) {
10112 if ((Subtarget->hasAVX512())) {
10113 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZrr, RC: &X86::VR128XRegClass, Op0);
10114 }
10115 return Register();
10116}
10117
10118Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Register Op0) {
10119 if ((Subtarget->hasAVX512())) {
10120 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZrr, RC: &X86::VR256XRegClass, Op0);
10121 }
10122 return Register();
10123}
10124
10125Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, Register Op0) {
10126switch (RetVT.SimpleTy) {
10127 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0);
10128 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0);
10129 default: return Register();
10130}
10131}
10132
10133Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Register Op0) {
10134 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10135 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
10136 }
10137 return Register();
10138}
10139
10140Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Register Op0) {
10141 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10142 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
10143 }
10144 return Register();
10145}
10146
10147Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Register Op0) {
10148 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10149 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
10150 }
10151 return Register();
10152}
10153
10154Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, Register Op0) {
10155switch (RetVT.SimpleTy) {
10156 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0);
10157 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0);
10158 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0);
10159 default: return Register();
10160}
10161}
10162
10163Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Register Op0) {
10164 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10165 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
10166 }
10167 return Register();
10168}
10169
10170Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Register Op0) {
10171 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10172 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
10173 }
10174 return Register();
10175}
10176
10177Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Register Op0) {
10178 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10179 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
10180 }
10181 return Register();
10182}
10183
10184Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, Register Op0) {
10185switch (RetVT.SimpleTy) {
10186 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0);
10187 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0);
10188 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0);
10189 default: return Register();
10190}
10191}
10192
10193Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Register Op0) {
10194 if ((Subtarget->hasAVX512())) {
10195 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZrr, RC: &X86::VR128XRegClass, Op0);
10196 }
10197 return Register();
10198}
10199
10200Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Register Op0) {
10201 if ((Subtarget->hasAVX512())) {
10202 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZrr, RC: &X86::VR128XRegClass, Op0);
10203 }
10204 return Register();
10205}
10206
10207Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Register Op0) {
10208 if ((Subtarget->hasAVX512())) {
10209 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZrr, RC: &X86::VR256XRegClass, Op0);
10210 }
10211 return Register();
10212}
10213
10214Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, Register Op0) {
10215switch (RetVT.SimpleTy) {
10216 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0);
10217 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0);
10218 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0);
10219 default: return Register();
10220}
10221}
10222
10223Register fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, Register Op0) {
10224 switch (VT.SimpleTy) {
10225 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0);
10226 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0);
10227 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0);
10228 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0);
10229 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0);
10230 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0);
10231 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0);
10232 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0);
10233 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0);
10234 default: return Register();
10235 }
10236}
10237
10238// FastEmit functions for X86ISD::VZEXT_MOVL.
10239
10240Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v8i16_r(MVT RetVT, Register Op0) {
10241 if (RetVT.SimpleTy != MVT::v8i16)
10242 return Register();
10243 if ((Subtarget->hasAVX10_2())) {
10244 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPWILo2PWIZrr, RC: &X86::VR128XRegClass, Op0);
10245 }
10246 return Register();
10247}
10248
10249Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v4i32_r(MVT RetVT, Register Op0) {
10250 if (RetVT.SimpleTy != MVT::v4i32)
10251 return Register();
10252 if ((Subtarget->hasAVX10_2())) {
10253 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPDILo2PDIZrr, RC: &X86::VR128XRegClass, Op0);
10254 }
10255 return Register();
10256}
10257
10258Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, Register Op0) {
10259 if (RetVT.SimpleTy != MVT::v2i64)
10260 return Register();
10261 if ((Subtarget->hasAVX512())) {
10262 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
10263 }
10264 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10265 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10266 }
10267 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10268 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10269 }
10270 return Register();
10271}
10272
10273Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, Register Op0) {
10274 if (RetVT.SimpleTy != MVT::v2f64)
10275 return Register();
10276 if ((Subtarget->hasAVX512())) {
10277 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
10278 }
10279 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10280 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10281 }
10282 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10283 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10284 }
10285 return Register();
10286}
10287
10288Register fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, Register Op0) {
10289 switch (VT.SimpleTy) {
10290 case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v8i16_r(RetVT, Op0);
10291 case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v4i32_r(RetVT, Op0);
10292 case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0);
10293 case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0);
10294 default: return Register();
10295 }
10296}
10297
10298// Top-level FastEmit function.
10299
10300Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
10301 switch (Opcode) {
10302 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
10303 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
10304 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
10305 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
10306 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
10307 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
10308 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
10309 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
10310 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
10311 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
10312 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
10313 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
10314 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
10315 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
10316 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
10317 case ISD::LLRINT: return fastEmit_ISD_LLRINT_r(VT, RetVT, Op0);
10318 case ISD::LRINT: return fastEmit_ISD_LRINT_r(VT, RetVT, Op0);
10319 case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0);
10320 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
10321 case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
10322 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
10323 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
10324 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
10325 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
10326 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
10327 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
10328 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
10329 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
10330 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
10331 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
10332 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
10333 case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
10334 case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0);
10335 case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0);
10336 case X86ISD::CVTNEPS2BF16: return fastEmit_X86ISD_CVTNEPS2BF16_r(VT, RetVT, Op0);
10337 case X86ISD::CVTP2IBS: return fastEmit_X86ISD_CVTP2IBS_r(VT, RetVT, Op0);
10338 case X86ISD::CVTP2IUBS: return fastEmit_X86ISD_CVTP2IUBS_r(VT, RetVT, Op0);
10339 case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0);
10340 case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0);
10341 case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0);
10342 case X86ISD::CVTPH2PS_SAE: return fastEmit_X86ISD_CVTPH2PS_SAE_r(VT, RetVT, Op0);
10343 case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0);
10344 case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0);
10345 case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0);
10346 case X86ISD::CVTTP2IBS: return fastEmit_X86ISD_CVTTP2IBS_r(VT, RetVT, Op0);
10347 case X86ISD::CVTTP2IBS_SAE: return fastEmit_X86ISD_CVTTP2IBS_SAE_r(VT, RetVT, Op0);
10348 case X86ISD::CVTTP2IUBS: return fastEmit_X86ISD_CVTTP2IUBS_r(VT, RetVT, Op0);
10349 case X86ISD::CVTTP2IUBS_SAE: return fastEmit_X86ISD_CVTTP2IUBS_SAE_r(VT, RetVT, Op0);
10350 case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0);
10351 case X86ISD::CVTTP2SIS: return fastEmit_X86ISD_CVTTP2SIS_r(VT, RetVT, Op0);
10352 case X86ISD::CVTTP2SIS_SAE: return fastEmit_X86ISD_CVTTP2SIS_SAE_r(VT, RetVT, Op0);
10353 case X86ISD::CVTTP2SI_SAE: return fastEmit_X86ISD_CVTTP2SI_SAE_r(VT, RetVT, Op0);
10354 case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0);
10355 case X86ISD::CVTTP2UIS: return fastEmit_X86ISD_CVTTP2UIS_r(VT, RetVT, Op0);
10356 case X86ISD::CVTTP2UIS_SAE: return fastEmit_X86ISD_CVTTP2UIS_SAE_r(VT, RetVT, Op0);
10357 case X86ISD::CVTTP2UI_SAE: return fastEmit_X86ISD_CVTTP2UI_SAE_r(VT, RetVT, Op0);
10358 case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0);
10359 case X86ISD::CVTTS2SIS: return fastEmit_X86ISD_CVTTS2SIS_r(VT, RetVT, Op0);
10360 case X86ISD::CVTTS2SIS_SAE: return fastEmit_X86ISD_CVTTS2SIS_SAE_r(VT, RetVT, Op0);
10361 case X86ISD::CVTTS2SI_SAE: return fastEmit_X86ISD_CVTTS2SI_SAE_r(VT, RetVT, Op0);
10362 case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0);
10363 case X86ISD::CVTTS2UIS: return fastEmit_X86ISD_CVTTS2UIS_r(VT, RetVT, Op0);
10364 case X86ISD::CVTTS2UIS_SAE: return fastEmit_X86ISD_CVTTS2UIS_SAE_r(VT, RetVT, Op0);
10365 case X86ISD::CVTTS2UI_SAE: return fastEmit_X86ISD_CVTTS2UI_SAE_r(VT, RetVT, Op0);
10366 case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0);
10367 case X86ISD::DYN_ALLOCA: return fastEmit_X86ISD_DYN_ALLOCA_r(VT, RetVT, Op0);
10368 case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0);
10369 case X86ISD::FGETEXP: return fastEmit_X86ISD_FGETEXP_r(VT, RetVT, Op0);
10370 case X86ISD::FGETEXP_SAE: return fastEmit_X86ISD_FGETEXP_SAE_r(VT, RetVT, Op0);
10371 case X86ISD::FP_TO_SINT_SAT: return fastEmit_X86ISD_FP_TO_SINT_SAT_r(VT, RetVT, Op0);
10372 case X86ISD::FP_TO_UINT_SAT: return fastEmit_X86ISD_FP_TO_UINT_SAT_r(VT, RetVT, Op0);
10373 case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0);
10374 case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0);
10375 case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0);
10376 case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0);
10377 case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0);
10378 case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0);
10379 case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0);
10380 case X86ISD::MOVQ2DQ: return fastEmit_X86ISD_MOVQ2DQ_r(VT, RetVT, Op0);
10381 case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0);
10382 case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0);
10383 case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0);
10384 case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0);
10385 case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0);
10386 case X86ISD::PROBED_ALLOCA: return fastEmit_X86ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
10387 case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0);
10388 case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0);
10389 case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0);
10390 case X86ISD::STRICT_CVTPH2PS: return fastEmit_X86ISD_STRICT_CVTPH2PS_r(VT, RetVT, Op0);
10391 case X86ISD::STRICT_CVTSI2P: return fastEmit_X86ISD_STRICT_CVTSI2P_r(VT, RetVT, Op0);
10392 case X86ISD::STRICT_CVTTP2SI: return fastEmit_X86ISD_STRICT_CVTTP2SI_r(VT, RetVT, Op0);
10393 case X86ISD::STRICT_CVTTP2UI: return fastEmit_X86ISD_STRICT_CVTTP2UI_r(VT, RetVT, Op0);
10394 case X86ISD::STRICT_CVTUI2P: return fastEmit_X86ISD_STRICT_CVTUI2P_r(VT, RetVT, Op0);
10395 case X86ISD::STRICT_VFPEXT: return fastEmit_X86ISD_STRICT_VFPEXT_r(VT, RetVT, Op0);
10396 case X86ISD::STRICT_VFPROUND: return fastEmit_X86ISD_STRICT_VFPROUND_r(VT, RetVT, Op0);
10397 case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0);
10398 case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0);
10399 case X86ISD::VCVTHF82PH: return fastEmit_X86ISD_VCVTHF82PH_r(VT, RetVT, Op0);
10400 case X86ISD::VCVTPH2BF8: return fastEmit_X86ISD_VCVTPH2BF8_r(VT, RetVT, Op0);
10401 case X86ISD::VCVTPH2BF8S: return fastEmit_X86ISD_VCVTPH2BF8S_r(VT, RetVT, Op0);
10402 case X86ISD::VCVTPH2HF8: return fastEmit_X86ISD_VCVTPH2HF8_r(VT, RetVT, Op0);
10403 case X86ISD::VCVTPH2HF8S: return fastEmit_X86ISD_VCVTPH2HF8S_r(VT, RetVT, Op0);
10404 case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0);
10405 case X86ISD::VFPEXT_SAE: return fastEmit_X86ISD_VFPEXT_SAE_r(VT, RetVT, Op0);
10406 case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0);
10407 case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0);
10408 case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0);
10409 case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0);
10410 case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0);
10411 default: return Register();
10412 }
10413}
10414
10415// FastEmit functions for ISD::ADD.
10416
10417Register fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
10418 if (RetVT.SimpleTy != MVT::i8)
10419 return Register();
10420 if ((Subtarget->hasNDD())) {
10421 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10422 }
10423 if ((!Subtarget->hasNDD())) {
10424 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr, RC: &X86::GR8RegClass, Op0, Op1);
10425 }
10426 return Register();
10427}
10428
10429Register fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
10430 if (RetVT.SimpleTy != MVT::i16)
10431 return Register();
10432 if ((Subtarget->hasNDD())) {
10433 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10434 }
10435 if ((!Subtarget->hasNDD())) {
10436 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr, RC: &X86::GR16RegClass, Op0, Op1);
10437 }
10438 return Register();
10439}
10440
10441Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10442 if (RetVT.SimpleTy != MVT::i32)
10443 return Register();
10444 if ((Subtarget->hasNDD())) {
10445 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10446 }
10447 if ((!Subtarget->hasNDD())) {
10448 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr, RC: &X86::GR32RegClass, Op0, Op1);
10449 }
10450 return Register();
10451}
10452
10453Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10454 if (RetVT.SimpleTy != MVT::i64)
10455 return Register();
10456 if ((Subtarget->hasNDD())) {
10457 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10458 }
10459 if ((!Subtarget->hasNDD())) {
10460 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr, RC: &X86::GR64RegClass, Op0, Op1);
10461 }
10462 return Register();
10463}
10464
10465Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10466 if (RetVT.SimpleTy != MVT::v16i8)
10467 return Register();
10468 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10469 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10470 }
10471 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10472 return fastEmitInst_rr(MachineInstOpcode: X86::PADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
10473 }
10474 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10475 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
10476 }
10477 return Register();
10478}
10479
10480Register fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10481 if (RetVT.SimpleTy != MVT::v32i8)
10482 return Register();
10483 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10484 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10485 }
10486 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10487 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10488 }
10489 return Register();
10490}
10491
10492Register fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10493 if (RetVT.SimpleTy != MVT::v64i8)
10494 return Register();
10495 if ((Subtarget->hasBWI())) {
10496 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10497 }
10498 return Register();
10499}
10500
10501Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10502 if (RetVT.SimpleTy != MVT::v8i16)
10503 return Register();
10504 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10505 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10506 }
10507 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10508 return fastEmitInst_rr(MachineInstOpcode: X86::PADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
10509 }
10510 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10511 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
10512 }
10513 return Register();
10514}
10515
10516Register fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10517 if (RetVT.SimpleTy != MVT::v16i16)
10518 return Register();
10519 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10520 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10521 }
10522 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10523 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10524 }
10525 return Register();
10526}
10527
10528Register fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10529 if (RetVT.SimpleTy != MVT::v32i16)
10530 return Register();
10531 if ((Subtarget->hasBWI())) {
10532 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10533 }
10534 return Register();
10535}
10536
10537Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10538 if (RetVT.SimpleTy != MVT::v4i32)
10539 return Register();
10540 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10541 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10542 }
10543 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10544 return fastEmitInst_rr(MachineInstOpcode: X86::PADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
10545 }
10546 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10547 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
10548 }
10549 return Register();
10550}
10551
10552Register fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
10553 if (RetVT.SimpleTy != MVT::v8i32)
10554 return Register();
10555 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10556 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10557 }
10558 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10559 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10560 }
10561 return Register();
10562}
10563
10564Register fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
10565 if (RetVT.SimpleTy != MVT::v16i32)
10566 return Register();
10567 if ((Subtarget->hasAVX512())) {
10568 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10569 }
10570 return Register();
10571}
10572
10573Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10574 if (RetVT.SimpleTy != MVT::v2i64)
10575 return Register();
10576 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10577 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10578 }
10579 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10580 return fastEmitInst_rr(MachineInstOpcode: X86::PADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
10581 }
10582 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10583 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
10584 }
10585 return Register();
10586}
10587
10588Register fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
10589 if (RetVT.SimpleTy != MVT::v4i64)
10590 return Register();
10591 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10592 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10593 }
10594 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10595 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
10596 }
10597 return Register();
10598}
10599
10600Register fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
10601 if (RetVT.SimpleTy != MVT::v8i64)
10602 return Register();
10603 if ((Subtarget->hasAVX512())) {
10604 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10605 }
10606 return Register();
10607}
10608
10609Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10610 switch (VT.SimpleTy) {
10611 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op1);
10612 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op1);
10613 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10614 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10615 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10616 case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op1);
10617 case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op1);
10618 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10619 case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op1);
10620 case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op1);
10621 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10622 case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op1);
10623 case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op1);
10624 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10625 case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op1);
10626 case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op1);
10627 default: return Register();
10628 }
10629}
10630
10631// FastEmit functions for ISD::AND.
10632
10633Register fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
10634 if (RetVT.SimpleTy != MVT::i8)
10635 return Register();
10636 if ((Subtarget->hasNDD())) {
10637 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10638 }
10639 if ((!Subtarget->hasNDD())) {
10640 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr, RC: &X86::GR8RegClass, Op0, Op1);
10641 }
10642 return Register();
10643}
10644
10645Register fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
10646 if (RetVT.SimpleTy != MVT::i16)
10647 return Register();
10648 if ((Subtarget->hasNDD())) {
10649 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10650 }
10651 if ((!Subtarget->hasNDD())) {
10652 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr, RC: &X86::GR16RegClass, Op0, Op1);
10653 }
10654 return Register();
10655}
10656
10657Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10658 if (RetVT.SimpleTy != MVT::i32)
10659 return Register();
10660 if ((Subtarget->hasNDD())) {
10661 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10662 }
10663 if ((!Subtarget->hasNDD())) {
10664 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr, RC: &X86::GR32RegClass, Op0, Op1);
10665 }
10666 return Register();
10667}
10668
10669Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10670 if (RetVT.SimpleTy != MVT::i64)
10671 return Register();
10672 if ((Subtarget->hasNDD())) {
10673 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10674 }
10675 if ((!Subtarget->hasNDD())) {
10676 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr, RC: &X86::GR64RegClass, Op0, Op1);
10677 }
10678 return Register();
10679}
10680
10681Register fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10682 if (RetVT.SimpleTy != MVT::v8i1)
10683 return Register();
10684 if ((Subtarget->hasDQI())) {
10685 return fastEmitInst_rr(MachineInstOpcode: X86::KANDBkk, RC: &X86::VK8RegClass, Op0, Op1);
10686 }
10687 return Register();
10688}
10689
10690Register fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
10691 if (RetVT.SimpleTy != MVT::v16i1)
10692 return Register();
10693 if ((Subtarget->hasAVX512())) {
10694 return fastEmitInst_rr(MachineInstOpcode: X86::KANDWkk, RC: &X86::VK16RegClass, Op0, Op1);
10695 }
10696 return Register();
10697}
10698
10699Register fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
10700 if (RetVT.SimpleTy != MVT::v32i1)
10701 return Register();
10702 if ((Subtarget->hasBWI())) {
10703 return fastEmitInst_rr(MachineInstOpcode: X86::KANDDkk, RC: &X86::VK32RegClass, Op0, Op1);
10704 }
10705 return Register();
10706}
10707
10708Register fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
10709 if (RetVT.SimpleTy != MVT::v64i1)
10710 return Register();
10711 if ((Subtarget->hasBWI())) {
10712 return fastEmitInst_rr(MachineInstOpcode: X86::KANDQkk, RC: &X86::VK64RegClass, Op0, Op1);
10713 }
10714 return Register();
10715}
10716
10717Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10718 if (RetVT.SimpleTy != MVT::v16i8)
10719 return Register();
10720 if ((Subtarget->hasVLX())) {
10721 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10722 }
10723 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10724 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10725 }
10726 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10727 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10728 }
10729 return Register();
10730}
10731
10732Register fastEmit_ISD_AND_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10733 if (RetVT.SimpleTy != MVT::v32i8)
10734 return Register();
10735 if ((Subtarget->hasVLX())) {
10736 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10737 }
10738 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10739 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10740 }
10741 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10742 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10743 }
10744 return Register();
10745}
10746
10747Register fastEmit_ISD_AND_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10748 if (RetVT.SimpleTy != MVT::v64i8)
10749 return Register();
10750 if ((Subtarget->hasAVX512())) {
10751 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10752 }
10753 return Register();
10754}
10755
10756Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10757 if (RetVT.SimpleTy != MVT::v8i16)
10758 return Register();
10759 if ((Subtarget->hasVLX())) {
10760 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10761 }
10762 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10763 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10764 }
10765 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10766 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10767 }
10768 return Register();
10769}
10770
10771Register fastEmit_ISD_AND_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10772 if (RetVT.SimpleTy != MVT::v16i16)
10773 return Register();
10774 if ((Subtarget->hasVLX())) {
10775 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10776 }
10777 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10778 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10779 }
10780 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10781 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10782 }
10783 return Register();
10784}
10785
10786Register fastEmit_ISD_AND_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10787 if (RetVT.SimpleTy != MVT::v32i16)
10788 return Register();
10789 if ((Subtarget->hasAVX512())) {
10790 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10791 }
10792 return Register();
10793}
10794
10795Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10796 if (RetVT.SimpleTy != MVT::v4i32)
10797 return Register();
10798 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10799 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10800 }
10801 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10802 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10803 }
10804 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10805 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10806 }
10807 return Register();
10808}
10809
10810Register fastEmit_ISD_AND_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
10811 if (RetVT.SimpleTy != MVT::v8i32)
10812 return Register();
10813 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10814 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10815 }
10816 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10817 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10818 }
10819 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10820 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10821 }
10822 return Register();
10823}
10824
10825Register fastEmit_ISD_AND_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
10826 if (RetVT.SimpleTy != MVT::v16i32)
10827 return Register();
10828 if ((Subtarget->hasAVX512())) {
10829 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10830 }
10831 return Register();
10832}
10833
10834Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10835 if (RetVT.SimpleTy != MVT::v2i64)
10836 return Register();
10837 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10838 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10839 }
10840 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10841 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10842 }
10843 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10844 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10845 }
10846 return Register();
10847}
10848
10849Register fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
10850 if (RetVT.SimpleTy != MVT::v4i64)
10851 return Register();
10852 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10853 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10854 }
10855 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10856 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10857 }
10858 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10859 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10860 }
10861 return Register();
10862}
10863
10864Register fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
10865 if (RetVT.SimpleTy != MVT::v8i64)
10866 return Register();
10867 if ((Subtarget->hasAVX512())) {
10868 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10869 }
10870 return Register();
10871}
10872
10873Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10874 switch (VT.SimpleTy) {
10875 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op1);
10876 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op1);
10877 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10878 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10879 case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op1);
10880 case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op1);
10881 case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op1);
10882 case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op1);
10883 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10884 case MVT::v32i8: return fastEmit_ISD_AND_MVT_v32i8_rr(RetVT, Op0, Op1);
10885 case MVT::v64i8: return fastEmit_ISD_AND_MVT_v64i8_rr(RetVT, Op0, Op1);
10886 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10887 case MVT::v16i16: return fastEmit_ISD_AND_MVT_v16i16_rr(RetVT, Op0, Op1);
10888 case MVT::v32i16: return fastEmit_ISD_AND_MVT_v32i16_rr(RetVT, Op0, Op1);
10889 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10890 case MVT::v8i32: return fastEmit_ISD_AND_MVT_v8i32_rr(RetVT, Op0, Op1);
10891 case MVT::v16i32: return fastEmit_ISD_AND_MVT_v16i32_rr(RetVT, Op0, Op1);
10892 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10893 case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op1);
10894 case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op1);
10895 default: return Register();
10896 }
10897}
10898
10899// FastEmit functions for ISD::AVGCEILU.
10900
10901Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10902 if (RetVT.SimpleTy != MVT::v16i8)
10903 return Register();
10904 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10905 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10906 }
10907 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10908 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
10909 }
10910 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10911 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
10912 }
10913 return Register();
10914}
10915
10916Register fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10917 if (RetVT.SimpleTy != MVT::v32i8)
10918 return Register();
10919 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10920 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10921 }
10922 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10923 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10924 }
10925 return Register();
10926}
10927
10928Register fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10929 if (RetVT.SimpleTy != MVT::v64i8)
10930 return Register();
10931 if ((Subtarget->hasBWI())) {
10932 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10933 }
10934 return Register();
10935}
10936
10937Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10938 if (RetVT.SimpleTy != MVT::v8i16)
10939 return Register();
10940 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10941 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10942 }
10943 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10944 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
10945 }
10946 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10947 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
10948 }
10949 return Register();
10950}
10951
10952Register fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10953 if (RetVT.SimpleTy != MVT::v16i16)
10954 return Register();
10955 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10956 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10957 }
10958 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10959 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10960 }
10961 return Register();
10962}
10963
10964Register fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10965 if (RetVT.SimpleTy != MVT::v32i16)
10966 return Register();
10967 if ((Subtarget->hasBWI())) {
10968 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10969 }
10970 return Register();
10971}
10972
10973Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10974 switch (VT.SimpleTy) {
10975 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10976 case MVT::v32i8: return fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(RetVT, Op0, Op1);
10977 case MVT::v64i8: return fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(RetVT, Op0, Op1);
10978 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10979 case MVT::v16i16: return fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(RetVT, Op0, Op1);
10980 case MVT::v32i16: return fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(RetVT, Op0, Op1);
10981 default: return Register();
10982 }
10983}
10984
10985// FastEmit functions for ISD::FADD.
10986
10987Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10988 if (RetVT.SimpleTy != MVT::f16)
10989 return Register();
10990 if ((Subtarget->hasFP16())) {
10991 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
10992 }
10993 return Register();
10994}
10995
10996Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10997 if (RetVT.SimpleTy != MVT::f32)
10998 return Register();
10999 if ((Subtarget->hasAVX512())) {
11000 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11001 }
11002 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11003 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11004 }
11005 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11006 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11007 }
11008 if ((!Subtarget->hasSSE1())) {
11009 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11010 }
11011 return Register();
11012}
11013
11014Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11015 if (RetVT.SimpleTy != MVT::f64)
11016 return Register();
11017 if ((Subtarget->hasAVX512())) {
11018 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11019 }
11020 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11021 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11022 }
11023 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11024 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11025 }
11026 if ((!Subtarget->hasSSE2())) {
11027 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11028 }
11029 return Register();
11030}
11031
11032Register fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11033 if (RetVT.SimpleTy != MVT::f80)
11034 return Register();
11035 if ((Subtarget->hasX87())) {
11036 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11037 }
11038 return Register();
11039}
11040
11041Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11042 if (RetVT.SimpleTy != MVT::v8f16)
11043 return Register();
11044 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11045 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11046 }
11047 return Register();
11048}
11049
11050Register fastEmit_ISD_FADD_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11051 if (RetVT.SimpleTy != MVT::v16f16)
11052 return Register();
11053 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11054 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11055 }
11056 return Register();
11057}
11058
11059Register fastEmit_ISD_FADD_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11060 if (RetVT.SimpleTy != MVT::v32f16)
11061 return Register();
11062 if ((Subtarget->hasFP16())) {
11063 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11064 }
11065 return Register();
11066}
11067
11068Register fastEmit_ISD_FADD_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11069 if (RetVT.SimpleTy != MVT::v8bf16)
11070 return Register();
11071 if ((Subtarget->hasAVX10_2())) {
11072 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11073 }
11074 return Register();
11075}
11076
11077Register fastEmit_ISD_FADD_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11078 if (RetVT.SimpleTy != MVT::v16bf16)
11079 return Register();
11080 if ((Subtarget->hasAVX10_2())) {
11081 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11082 }
11083 return Register();
11084}
11085
11086Register fastEmit_ISD_FADD_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11087 if (RetVT.SimpleTy != MVT::v32bf16)
11088 return Register();
11089 if ((Subtarget->hasAVX10_2())) {
11090 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11091 }
11092 return Register();
11093}
11094
11095Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11096 if (RetVT.SimpleTy != MVT::v4f32)
11097 return Register();
11098 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11099 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11100 }
11101 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11102 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11103 }
11104 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11105 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11106 }
11107 return Register();
11108}
11109
11110Register fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11111 if (RetVT.SimpleTy != MVT::v8f32)
11112 return Register();
11113 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11114 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11115 }
11116 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11117 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11118 }
11119 return Register();
11120}
11121
11122Register fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11123 if (RetVT.SimpleTy != MVT::v16f32)
11124 return Register();
11125 if ((Subtarget->hasAVX512())) {
11126 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11127 }
11128 return Register();
11129}
11130
11131Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11132 if (RetVT.SimpleTy != MVT::v2f64)
11133 return Register();
11134 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11135 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11136 }
11137 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11138 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11139 }
11140 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11141 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11142 }
11143 return Register();
11144}
11145
11146Register fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11147 if (RetVT.SimpleTy != MVT::v4f64)
11148 return Register();
11149 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11150 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11151 }
11152 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11153 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11154 }
11155 return Register();
11156}
11157
11158Register fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11159 if (RetVT.SimpleTy != MVT::v8f64)
11160 return Register();
11161 if ((Subtarget->hasAVX512())) {
11162 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11163 }
11164 return Register();
11165}
11166
11167Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11168 switch (VT.SimpleTy) {
11169 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11170 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11171 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11172 case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op1);
11173 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11174 case MVT::v16f16: return fastEmit_ISD_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
11175 case MVT::v32f16: return fastEmit_ISD_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
11176 case MVT::v8bf16: return fastEmit_ISD_FADD_MVT_v8bf16_rr(RetVT, Op0, Op1);
11177 case MVT::v16bf16: return fastEmit_ISD_FADD_MVT_v16bf16_rr(RetVT, Op0, Op1);
11178 case MVT::v32bf16: return fastEmit_ISD_FADD_MVT_v32bf16_rr(RetVT, Op0, Op1);
11179 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11180 case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
11181 case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
11182 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11183 case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
11184 case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
11185 default: return Register();
11186 }
11187}
11188
11189// FastEmit functions for ISD::FDIV.
11190
11191Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11192 if (RetVT.SimpleTy != MVT::f16)
11193 return Register();
11194 if ((Subtarget->hasFP16())) {
11195 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11196 }
11197 return Register();
11198}
11199
11200Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11201 if (RetVT.SimpleTy != MVT::f32)
11202 return Register();
11203 if ((Subtarget->hasAVX512())) {
11204 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11205 }
11206 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11207 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11208 }
11209 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11210 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11211 }
11212 if ((!Subtarget->hasSSE1())) {
11213 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11214 }
11215 return Register();
11216}
11217
11218Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11219 if (RetVT.SimpleTy != MVT::f64)
11220 return Register();
11221 if ((Subtarget->hasAVX512())) {
11222 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11223 }
11224 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11225 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11226 }
11227 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11228 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11229 }
11230 if ((!Subtarget->hasSSE2())) {
11231 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11232 }
11233 return Register();
11234}
11235
11236Register fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11237 if (RetVT.SimpleTy != MVT::f80)
11238 return Register();
11239 if ((Subtarget->hasX87())) {
11240 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11241 }
11242 return Register();
11243}
11244
11245Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11246 if (RetVT.SimpleTy != MVT::v8f16)
11247 return Register();
11248 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11249 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11250 }
11251 return Register();
11252}
11253
11254Register fastEmit_ISD_FDIV_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11255 if (RetVT.SimpleTy != MVT::v16f16)
11256 return Register();
11257 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11258 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11259 }
11260 return Register();
11261}
11262
11263Register fastEmit_ISD_FDIV_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11264 if (RetVT.SimpleTy != MVT::v32f16)
11265 return Register();
11266 if ((Subtarget->hasFP16())) {
11267 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11268 }
11269 return Register();
11270}
11271
11272Register fastEmit_ISD_FDIV_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11273 if (RetVT.SimpleTy != MVT::v8bf16)
11274 return Register();
11275 if ((Subtarget->hasAVX10_2())) {
11276 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11277 }
11278 return Register();
11279}
11280
11281Register fastEmit_ISD_FDIV_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11282 if (RetVT.SimpleTy != MVT::v16bf16)
11283 return Register();
11284 if ((Subtarget->hasAVX10_2())) {
11285 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11286 }
11287 return Register();
11288}
11289
11290Register fastEmit_ISD_FDIV_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11291 if (RetVT.SimpleTy != MVT::v32bf16)
11292 return Register();
11293 if ((Subtarget->hasAVX10_2())) {
11294 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11295 }
11296 return Register();
11297}
11298
11299Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11300 if (RetVT.SimpleTy != MVT::v4f32)
11301 return Register();
11302 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11303 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11304 }
11305 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11306 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11307 }
11308 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11309 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11310 }
11311 return Register();
11312}
11313
11314Register fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11315 if (RetVT.SimpleTy != MVT::v8f32)
11316 return Register();
11317 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11318 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11319 }
11320 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11321 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11322 }
11323 return Register();
11324}
11325
11326Register fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11327 if (RetVT.SimpleTy != MVT::v16f32)
11328 return Register();
11329 if ((Subtarget->hasAVX512())) {
11330 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11331 }
11332 return Register();
11333}
11334
11335Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11336 if (RetVT.SimpleTy != MVT::v2f64)
11337 return Register();
11338 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11339 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11340 }
11341 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11342 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11343 }
11344 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11345 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11346 }
11347 return Register();
11348}
11349
11350Register fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11351 if (RetVT.SimpleTy != MVT::v4f64)
11352 return Register();
11353 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11354 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11355 }
11356 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11357 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11358 }
11359 return Register();
11360}
11361
11362Register fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11363 if (RetVT.SimpleTy != MVT::v8f64)
11364 return Register();
11365 if ((Subtarget->hasAVX512())) {
11366 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11367 }
11368 return Register();
11369}
11370
11371Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11372 switch (VT.SimpleTy) {
11373 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11374 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11375 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11376 case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
11377 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11378 case MVT::v16f16: return fastEmit_ISD_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
11379 case MVT::v32f16: return fastEmit_ISD_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
11380 case MVT::v8bf16: return fastEmit_ISD_FDIV_MVT_v8bf16_rr(RetVT, Op0, Op1);
11381 case MVT::v16bf16: return fastEmit_ISD_FDIV_MVT_v16bf16_rr(RetVT, Op0, Op1);
11382 case MVT::v32bf16: return fastEmit_ISD_FDIV_MVT_v32bf16_rr(RetVT, Op0, Op1);
11383 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11384 case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
11385 case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
11386 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11387 case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
11388 case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
11389 default: return Register();
11390 }
11391}
11392
11393// FastEmit functions for ISD::FMUL.
11394
11395Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11396 if (RetVT.SimpleTy != MVT::f16)
11397 return Register();
11398 if ((Subtarget->hasFP16())) {
11399 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11400 }
11401 return Register();
11402}
11403
11404Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11405 if (RetVT.SimpleTy != MVT::f32)
11406 return Register();
11407 if ((Subtarget->hasAVX512())) {
11408 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11409 }
11410 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11411 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11412 }
11413 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11414 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11415 }
11416 if ((!Subtarget->hasSSE1())) {
11417 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11418 }
11419 return Register();
11420}
11421
11422Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11423 if (RetVT.SimpleTy != MVT::f64)
11424 return Register();
11425 if ((Subtarget->hasAVX512())) {
11426 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11427 }
11428 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11429 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11430 }
11431 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11432 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11433 }
11434 if ((!Subtarget->hasSSE2())) {
11435 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11436 }
11437 return Register();
11438}
11439
11440Register fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11441 if (RetVT.SimpleTy != MVT::f80)
11442 return Register();
11443 if ((Subtarget->hasX87())) {
11444 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11445 }
11446 return Register();
11447}
11448
11449Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11450 if (RetVT.SimpleTy != MVT::v8f16)
11451 return Register();
11452 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11453 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11454 }
11455 return Register();
11456}
11457
11458Register fastEmit_ISD_FMUL_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11459 if (RetVT.SimpleTy != MVT::v16f16)
11460 return Register();
11461 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11462 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11463 }
11464 return Register();
11465}
11466
11467Register fastEmit_ISD_FMUL_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11468 if (RetVT.SimpleTy != MVT::v32f16)
11469 return Register();
11470 if ((Subtarget->hasFP16())) {
11471 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11472 }
11473 return Register();
11474}
11475
11476Register fastEmit_ISD_FMUL_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11477 if (RetVT.SimpleTy != MVT::v8bf16)
11478 return Register();
11479 if ((Subtarget->hasAVX10_2())) {
11480 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11481 }
11482 return Register();
11483}
11484
11485Register fastEmit_ISD_FMUL_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11486 if (RetVT.SimpleTy != MVT::v16bf16)
11487 return Register();
11488 if ((Subtarget->hasAVX10_2())) {
11489 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11490 }
11491 return Register();
11492}
11493
11494Register fastEmit_ISD_FMUL_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11495 if (RetVT.SimpleTy != MVT::v32bf16)
11496 return Register();
11497 if ((Subtarget->hasAVX10_2())) {
11498 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11499 }
11500 return Register();
11501}
11502
11503Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11504 if (RetVT.SimpleTy != MVT::v4f32)
11505 return Register();
11506 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11507 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11508 }
11509 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11510 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11511 }
11512 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11513 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11514 }
11515 return Register();
11516}
11517
11518Register fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11519 if (RetVT.SimpleTy != MVT::v8f32)
11520 return Register();
11521 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11522 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11523 }
11524 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11525 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11526 }
11527 return Register();
11528}
11529
11530Register fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11531 if (RetVT.SimpleTy != MVT::v16f32)
11532 return Register();
11533 if ((Subtarget->hasAVX512())) {
11534 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11535 }
11536 return Register();
11537}
11538
11539Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11540 if (RetVT.SimpleTy != MVT::v2f64)
11541 return Register();
11542 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11543 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11544 }
11545 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11546 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11547 }
11548 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11549 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11550 }
11551 return Register();
11552}
11553
11554Register fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11555 if (RetVT.SimpleTy != MVT::v4f64)
11556 return Register();
11557 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11558 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11559 }
11560 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11561 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11562 }
11563 return Register();
11564}
11565
11566Register fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11567 if (RetVT.SimpleTy != MVT::v8f64)
11568 return Register();
11569 if ((Subtarget->hasAVX512())) {
11570 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11571 }
11572 return Register();
11573}
11574
11575Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11576 switch (VT.SimpleTy) {
11577 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11578 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11579 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11580 case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
11581 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11582 case MVT::v16f16: return fastEmit_ISD_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
11583 case MVT::v32f16: return fastEmit_ISD_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
11584 case MVT::v8bf16: return fastEmit_ISD_FMUL_MVT_v8bf16_rr(RetVT, Op0, Op1);
11585 case MVT::v16bf16: return fastEmit_ISD_FMUL_MVT_v16bf16_rr(RetVT, Op0, Op1);
11586 case MVT::v32bf16: return fastEmit_ISD_FMUL_MVT_v32bf16_rr(RetVT, Op0, Op1);
11587 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11588 case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
11589 case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
11590 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11591 case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
11592 case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
11593 default: return Register();
11594 }
11595}
11596
11597// FastEmit functions for ISD::FSUB.
11598
11599Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11600 if (RetVT.SimpleTy != MVT::f16)
11601 return Register();
11602 if ((Subtarget->hasFP16())) {
11603 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11604 }
11605 return Register();
11606}
11607
11608Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11609 if (RetVT.SimpleTy != MVT::f32)
11610 return Register();
11611 if ((Subtarget->hasAVX512())) {
11612 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11613 }
11614 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11615 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11616 }
11617 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11618 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11619 }
11620 if ((!Subtarget->hasSSE1())) {
11621 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11622 }
11623 return Register();
11624}
11625
11626Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11627 if (RetVT.SimpleTy != MVT::f64)
11628 return Register();
11629 if ((Subtarget->hasAVX512())) {
11630 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11631 }
11632 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11633 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11634 }
11635 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11636 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11637 }
11638 if ((!Subtarget->hasSSE2())) {
11639 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11640 }
11641 return Register();
11642}
11643
11644Register fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11645 if (RetVT.SimpleTy != MVT::f80)
11646 return Register();
11647 if ((Subtarget->hasX87())) {
11648 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11649 }
11650 return Register();
11651}
11652
11653Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11654 if (RetVT.SimpleTy != MVT::v8f16)
11655 return Register();
11656 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11657 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11658 }
11659 return Register();
11660}
11661
11662Register fastEmit_ISD_FSUB_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11663 if (RetVT.SimpleTy != MVT::v16f16)
11664 return Register();
11665 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11666 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11667 }
11668 return Register();
11669}
11670
11671Register fastEmit_ISD_FSUB_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11672 if (RetVT.SimpleTy != MVT::v32f16)
11673 return Register();
11674 if ((Subtarget->hasFP16())) {
11675 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11676 }
11677 return Register();
11678}
11679
11680Register fastEmit_ISD_FSUB_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11681 if (RetVT.SimpleTy != MVT::v8bf16)
11682 return Register();
11683 if ((Subtarget->hasAVX10_2())) {
11684 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11685 }
11686 return Register();
11687}
11688
11689Register fastEmit_ISD_FSUB_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11690 if (RetVT.SimpleTy != MVT::v16bf16)
11691 return Register();
11692 if ((Subtarget->hasAVX10_2())) {
11693 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11694 }
11695 return Register();
11696}
11697
11698Register fastEmit_ISD_FSUB_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11699 if (RetVT.SimpleTy != MVT::v32bf16)
11700 return Register();
11701 if ((Subtarget->hasAVX10_2())) {
11702 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11703 }
11704 return Register();
11705}
11706
11707Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11708 if (RetVT.SimpleTy != MVT::v4f32)
11709 return Register();
11710 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11711 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11712 }
11713 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11714 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11715 }
11716 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11717 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11718 }
11719 return Register();
11720}
11721
11722Register fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11723 if (RetVT.SimpleTy != MVT::v8f32)
11724 return Register();
11725 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11726 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11727 }
11728 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11729 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11730 }
11731 return Register();
11732}
11733
11734Register fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11735 if (RetVT.SimpleTy != MVT::v16f32)
11736 return Register();
11737 if ((Subtarget->hasAVX512())) {
11738 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11739 }
11740 return Register();
11741}
11742
11743Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11744 if (RetVT.SimpleTy != MVT::v2f64)
11745 return Register();
11746 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11747 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11748 }
11749 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11750 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11751 }
11752 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11753 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11754 }
11755 return Register();
11756}
11757
11758Register fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11759 if (RetVT.SimpleTy != MVT::v4f64)
11760 return Register();
11761 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11762 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11763 }
11764 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11765 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11766 }
11767 return Register();
11768}
11769
11770Register fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11771 if (RetVT.SimpleTy != MVT::v8f64)
11772 return Register();
11773 if ((Subtarget->hasAVX512())) {
11774 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11775 }
11776 return Register();
11777}
11778
11779Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11780 switch (VT.SimpleTy) {
11781 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11782 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11783 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11784 case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
11785 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11786 case MVT::v16f16: return fastEmit_ISD_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
11787 case MVT::v32f16: return fastEmit_ISD_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
11788 case MVT::v8bf16: return fastEmit_ISD_FSUB_MVT_v8bf16_rr(RetVT, Op0, Op1);
11789 case MVT::v16bf16: return fastEmit_ISD_FSUB_MVT_v16bf16_rr(RetVT, Op0, Op1);
11790 case MVT::v32bf16: return fastEmit_ISD_FSUB_MVT_v32bf16_rr(RetVT, Op0, Op1);
11791 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11792 case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
11793 case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
11794 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11795 case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
11796 case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
11797 default: return Register();
11798 }
11799}
11800
11801// FastEmit functions for ISD::MUL.
11802
11803Register fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
11804 if (RetVT.SimpleTy != MVT::i8)
11805 return Register();
11806 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::AL).addReg(RegNo: Op0);
11807 return fastEmitInst_r(MachineInstOpcode: X86::MUL8r, RC: &X86::GR8RegClass, Op0: Op1);
11808}
11809
11810Register fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
11811 if (RetVT.SimpleTy != MVT::i16)
11812 return Register();
11813 if ((Subtarget->hasNDD())) {
11814 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
11815 }
11816 if ((!Subtarget->hasNDD())) {
11817 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr, RC: &X86::GR16RegClass, Op0, Op1);
11818 }
11819 return Register();
11820}
11821
11822Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11823 if (RetVT.SimpleTy != MVT::i32)
11824 return Register();
11825 if ((Subtarget->hasNDD())) {
11826 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
11827 }
11828 if ((!Subtarget->hasNDD())) {
11829 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr, RC: &X86::GR32RegClass, Op0, Op1);
11830 }
11831 return Register();
11832}
11833
11834Register fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11835 if (RetVT.SimpleTy != MVT::i64)
11836 return Register();
11837 if ((Subtarget->hasNDD())) {
11838 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
11839 }
11840 if ((!Subtarget->hasNDD())) {
11841 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr, RC: &X86::GR64RegClass, Op0, Op1);
11842 }
11843 return Register();
11844}
11845
11846Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11847 if (RetVT.SimpleTy != MVT::v8i16)
11848 return Register();
11849 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11850 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11851 }
11852 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11853 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
11854 }
11855 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11856 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
11857 }
11858 return Register();
11859}
11860
11861Register fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
11862 if (RetVT.SimpleTy != MVT::v16i16)
11863 return Register();
11864 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11865 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11866 }
11867 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11868 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWYrr, RC: &X86::VR256RegClass, Op0, Op1);
11869 }
11870 return Register();
11871}
11872
11873Register fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
11874 if (RetVT.SimpleTy != MVT::v32i16)
11875 return Register();
11876 if ((Subtarget->hasBWI())) {
11877 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11878 }
11879 return Register();
11880}
11881
11882Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11883 if (RetVT.SimpleTy != MVT::v4i32)
11884 return Register();
11885 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11886 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11887 }
11888 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
11889 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
11890 }
11891 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11892 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
11893 }
11894 return Register();
11895}
11896
11897Register fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
11898 if (RetVT.SimpleTy != MVT::v8i32)
11899 return Register();
11900 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11901 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11902 }
11903 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11904 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11905 }
11906 return Register();
11907}
11908
11909Register fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
11910 if (RetVT.SimpleTy != MVT::v16i32)
11911 return Register();
11912 if ((Subtarget->hasAVX512())) {
11913 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11914 }
11915 return Register();
11916}
11917
11918Register fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11919 if (RetVT.SimpleTy != MVT::v2i64)
11920 return Register();
11921 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
11922 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11923 }
11924 return Register();
11925}
11926
11927Register fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
11928 if (RetVT.SimpleTy != MVT::v4i64)
11929 return Register();
11930 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
11931 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11932 }
11933 return Register();
11934}
11935
11936Register fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
11937 if (RetVT.SimpleTy != MVT::v8i64)
11938 return Register();
11939 if ((Subtarget->hasDQI())) {
11940 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZrr, RC: &X86::VR512RegClass, Op0, Op1);
11941 }
11942 return Register();
11943}
11944
11945Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11946 switch (VT.SimpleTy) {
11947 case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op1);
11948 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op1);
11949 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
11950 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op1);
11951 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
11952 case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op1);
11953 case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op1);
11954 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
11955 case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op1);
11956 case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op1);
11957 case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op1);
11958 case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op1);
11959 case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op1);
11960 default: return Register();
11961 }
11962}
11963
11964// FastEmit functions for ISD::MULHS.
11965
11966Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11967 if (RetVT.SimpleTy != MVT::v8i16)
11968 return Register();
11969 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11970 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11971 }
11972 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11973 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
11974 }
11975 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11976 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
11977 }
11978 return Register();
11979}
11980
11981Register fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
11982 if (RetVT.SimpleTy != MVT::v16i16)
11983 return Register();
11984 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11985 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11986 }
11987 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11988 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWYrr, RC: &X86::VR256RegClass, Op0, Op1);
11989 }
11990 return Register();
11991}
11992
11993Register fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
11994 if (RetVT.SimpleTy != MVT::v32i16)
11995 return Register();
11996 if ((Subtarget->hasBWI())) {
11997 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11998 }
11999 return Register();
12000}
12001
12002Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12003 switch (VT.SimpleTy) {
12004 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
12005 case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op1);
12006 case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op1);
12007 default: return Register();
12008 }
12009}
12010
12011// FastEmit functions for ISD::MULHU.
12012
12013Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12014 if (RetVT.SimpleTy != MVT::v8i16)
12015 return Register();
12016 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12017 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12018 }
12019 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12020 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12021 }
12022 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12023 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12024 }
12025 return Register();
12026}
12027
12028Register fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12029 if (RetVT.SimpleTy != MVT::v16i16)
12030 return Register();
12031 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12032 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12033 }
12034 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12035 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12036 }
12037 return Register();
12038}
12039
12040Register fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12041 if (RetVT.SimpleTy != MVT::v32i16)
12042 return Register();
12043 if ((Subtarget->hasBWI())) {
12044 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12045 }
12046 return Register();
12047}
12048
12049Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12050 switch (VT.SimpleTy) {
12051 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
12052 case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op1);
12053 case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op1);
12054 default: return Register();
12055 }
12056}
12057
12058// FastEmit functions for ISD::OR.
12059
12060Register fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12061 if (RetVT.SimpleTy != MVT::i8)
12062 return Register();
12063 if ((Subtarget->hasNDD())) {
12064 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
12065 }
12066 if ((!Subtarget->hasNDD())) {
12067 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr, RC: &X86::GR8RegClass, Op0, Op1);
12068 }
12069 return Register();
12070}
12071
12072Register fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
12073 if (RetVT.SimpleTy != MVT::i16)
12074 return Register();
12075 if ((Subtarget->hasNDD())) {
12076 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
12077 }
12078 if ((!Subtarget->hasNDD())) {
12079 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr, RC: &X86::GR16RegClass, Op0, Op1);
12080 }
12081 return Register();
12082}
12083
12084Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12085 if (RetVT.SimpleTy != MVT::i32)
12086 return Register();
12087 if ((Subtarget->hasNDD())) {
12088 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
12089 }
12090 if ((!Subtarget->hasNDD())) {
12091 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr, RC: &X86::GR32RegClass, Op0, Op1);
12092 }
12093 return Register();
12094}
12095
12096Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12097 if (RetVT.SimpleTy != MVT::i64)
12098 return Register();
12099 if ((Subtarget->hasNDD())) {
12100 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
12101 }
12102 if ((!Subtarget->hasNDD())) {
12103 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr, RC: &X86::GR64RegClass, Op0, Op1);
12104 }
12105 return Register();
12106}
12107
12108Register fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
12109 if (RetVT.SimpleTy != MVT::v8i1)
12110 return Register();
12111 if ((Subtarget->hasDQI())) {
12112 return fastEmitInst_rr(MachineInstOpcode: X86::KORBkk, RC: &X86::VK8RegClass, Op0, Op1);
12113 }
12114 return Register();
12115}
12116
12117Register fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
12118 if (RetVT.SimpleTy != MVT::v16i1)
12119 return Register();
12120 if ((Subtarget->hasAVX512())) {
12121 return fastEmitInst_rr(MachineInstOpcode: X86::KORWkk, RC: &X86::VK16RegClass, Op0, Op1);
12122 }
12123 return Register();
12124}
12125
12126Register fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
12127 if (RetVT.SimpleTy != MVT::v32i1)
12128 return Register();
12129 if ((Subtarget->hasBWI())) {
12130 return fastEmitInst_rr(MachineInstOpcode: X86::KORDkk, RC: &X86::VK32RegClass, Op0, Op1);
12131 }
12132 return Register();
12133}
12134
12135Register fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
12136 if (RetVT.SimpleTy != MVT::v64i1)
12137 return Register();
12138 if ((Subtarget->hasBWI())) {
12139 return fastEmitInst_rr(MachineInstOpcode: X86::KORQkk, RC: &X86::VK64RegClass, Op0, Op1);
12140 }
12141 return Register();
12142}
12143
12144Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12145 if (RetVT.SimpleTy != MVT::v16i8)
12146 return Register();
12147 if ((Subtarget->hasVLX())) {
12148 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12149 }
12150 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12151 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12152 }
12153 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12154 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12155 }
12156 return Register();
12157}
12158
12159Register fastEmit_ISD_OR_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12160 if (RetVT.SimpleTy != MVT::v32i8)
12161 return Register();
12162 if ((Subtarget->hasVLX())) {
12163 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12164 }
12165 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12166 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12167 }
12168 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12169 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12170 }
12171 return Register();
12172}
12173
12174Register fastEmit_ISD_OR_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12175 if (RetVT.SimpleTy != MVT::v64i8)
12176 return Register();
12177 if ((Subtarget->hasAVX512())) {
12178 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12179 }
12180 return Register();
12181}
12182
12183Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12184 if (RetVT.SimpleTy != MVT::v8i16)
12185 return Register();
12186 if ((Subtarget->hasVLX())) {
12187 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12188 }
12189 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12190 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12191 }
12192 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12193 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12194 }
12195 return Register();
12196}
12197
12198Register fastEmit_ISD_OR_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12199 if (RetVT.SimpleTy != MVT::v16i16)
12200 return Register();
12201 if ((Subtarget->hasVLX())) {
12202 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12203 }
12204 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12205 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12206 }
12207 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12208 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12209 }
12210 return Register();
12211}
12212
12213Register fastEmit_ISD_OR_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12214 if (RetVT.SimpleTy != MVT::v32i16)
12215 return Register();
12216 if ((Subtarget->hasAVX512())) {
12217 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12218 }
12219 return Register();
12220}
12221
12222Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12223 if (RetVT.SimpleTy != MVT::v4i32)
12224 return Register();
12225 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12226 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12227 }
12228 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12229 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12230 }
12231 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12232 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12233 }
12234 return Register();
12235}
12236
12237Register fastEmit_ISD_OR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12238 if (RetVT.SimpleTy != MVT::v8i32)
12239 return Register();
12240 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12241 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12242 }
12243 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12244 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12245 }
12246 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12247 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12248 }
12249 return Register();
12250}
12251
12252Register fastEmit_ISD_OR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12253 if (RetVT.SimpleTy != MVT::v16i32)
12254 return Register();
12255 if ((Subtarget->hasAVX512())) {
12256 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12257 }
12258 return Register();
12259}
12260
12261Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12262 if (RetVT.SimpleTy != MVT::v2i64)
12263 return Register();
12264 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12265 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12266 }
12267 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12268 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12269 }
12270 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12271 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12272 }
12273 return Register();
12274}
12275
12276Register fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12277 if (RetVT.SimpleTy != MVT::v4i64)
12278 return Register();
12279 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12280 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12281 }
12282 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12283 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12284 }
12285 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12286 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12287 }
12288 return Register();
12289}
12290
12291Register fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12292 if (RetVT.SimpleTy != MVT::v8i64)
12293 return Register();
12294 if ((Subtarget->hasAVX512())) {
12295 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12296 }
12297 return Register();
12298}
12299
12300Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12301 switch (VT.SimpleTy) {
12302 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op1);
12303 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op1);
12304 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12305 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12306 case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op1);
12307 case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op1);
12308 case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op1);
12309 case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op1);
12310 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12311 case MVT::v32i8: return fastEmit_ISD_OR_MVT_v32i8_rr(RetVT, Op0, Op1);
12312 case MVT::v64i8: return fastEmit_ISD_OR_MVT_v64i8_rr(RetVT, Op0, Op1);
12313 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12314 case MVT::v16i16: return fastEmit_ISD_OR_MVT_v16i16_rr(RetVT, Op0, Op1);
12315 case MVT::v32i16: return fastEmit_ISD_OR_MVT_v32i16_rr(RetVT, Op0, Op1);
12316 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12317 case MVT::v8i32: return fastEmit_ISD_OR_MVT_v8i32_rr(RetVT, Op0, Op1);
12318 case MVT::v16i32: return fastEmit_ISD_OR_MVT_v16i32_rr(RetVT, Op0, Op1);
12319 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12320 case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op1);
12321 case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op1);
12322 default: return Register();
12323 }
12324}
12325
12326// FastEmit functions for ISD::PDEP.
12327
12328Register fastEmit_ISD_PDEP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12329 if (RetVT.SimpleTy != MVT::i32)
12330 return Register();
12331 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12332 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
12333 }
12334 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12335 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr, RC: &X86::GR32RegClass, Op0, Op1);
12336 }
12337 return Register();
12338}
12339
12340Register fastEmit_ISD_PDEP_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12341 if (RetVT.SimpleTy != MVT::i64)
12342 return Register();
12343 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12344 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
12345 }
12346 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12347 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr, RC: &X86::GR64RegClass, Op0, Op1);
12348 }
12349 return Register();
12350}
12351
12352Register fastEmit_ISD_PDEP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12353 switch (VT.SimpleTy) {
12354 case MVT::i32: return fastEmit_ISD_PDEP_MVT_i32_rr(RetVT, Op0, Op1);
12355 case MVT::i64: return fastEmit_ISD_PDEP_MVT_i64_rr(RetVT, Op0, Op1);
12356 default: return Register();
12357 }
12358}
12359
12360// FastEmit functions for ISD::PEXT.
12361
12362Register fastEmit_ISD_PEXT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12363 if (RetVT.SimpleTy != MVT::i32)
12364 return Register();
12365 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12366 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
12367 }
12368 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12369 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr, RC: &X86::GR32RegClass, Op0, Op1);
12370 }
12371 return Register();
12372}
12373
12374Register fastEmit_ISD_PEXT_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12375 if (RetVT.SimpleTy != MVT::i64)
12376 return Register();
12377 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12378 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
12379 }
12380 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12381 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr, RC: &X86::GR64RegClass, Op0, Op1);
12382 }
12383 return Register();
12384}
12385
12386Register fastEmit_ISD_PEXT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12387 switch (VT.SimpleTy) {
12388 case MVT::i32: return fastEmit_ISD_PEXT_MVT_i32_rr(RetVT, Op0, Op1);
12389 case MVT::i64: return fastEmit_ISD_PEXT_MVT_i64_rr(RetVT, Op0, Op1);
12390 default: return Register();
12391 }
12392}
12393
12394// FastEmit functions for ISD::ROTL.
12395
12396Register fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12397 if (RetVT.SimpleTy != MVT::i8)
12398 return Register();
12399 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12400 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12401 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL_ND, RC: &X86::GR8RegClass, Op0);
12402 }
12403 if ((!Subtarget->hasNDD())) {
12404 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12405 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL, RC: &X86::GR8RegClass, Op0);
12406 }
12407 return Register();
12408}
12409
12410Register fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12411 if (RetVT.SimpleTy != MVT::v16i8)
12412 return Register();
12413 if ((Subtarget->hasXOP())) {
12414 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTBrr, RC: &X86::VR128RegClass, Op0, Op1);
12415 }
12416 return Register();
12417}
12418
12419Register fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12420 if (RetVT.SimpleTy != MVT::v8i16)
12421 return Register();
12422 if ((Subtarget->hasXOP())) {
12423 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTWrr, RC: &X86::VR128RegClass, Op0, Op1);
12424 }
12425 return Register();
12426}
12427
12428Register fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12429 if (RetVT.SimpleTy != MVT::v4i32)
12430 return Register();
12431 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12432 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12433 }
12434 if ((Subtarget->hasXOP())) {
12435 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTDrr, RC: &X86::VR128RegClass, Op0, Op1);
12436 }
12437 return Register();
12438}
12439
12440Register fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12441 if (RetVT.SimpleTy != MVT::v8i32)
12442 return Register();
12443 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12444 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12445 }
12446 return Register();
12447}
12448
12449Register fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12450 if (RetVT.SimpleTy != MVT::v16i32)
12451 return Register();
12452 if ((Subtarget->hasAVX512())) {
12453 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12454 }
12455 return Register();
12456}
12457
12458Register fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12459 if (RetVT.SimpleTy != MVT::v2i64)
12460 return Register();
12461 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12462 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12463 }
12464 if ((Subtarget->hasXOP())) {
12465 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTQrr, RC: &X86::VR128RegClass, Op0, Op1);
12466 }
12467 return Register();
12468}
12469
12470Register fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12471 if (RetVT.SimpleTy != MVT::v4i64)
12472 return Register();
12473 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12474 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12475 }
12476 return Register();
12477}
12478
12479Register fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12480 if (RetVT.SimpleTy != MVT::v8i64)
12481 return Register();
12482 if ((Subtarget->hasAVX512())) {
12483 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12484 }
12485 return Register();
12486}
12487
12488Register fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12489 switch (VT.SimpleTy) {
12490 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op1);
12491 case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op1);
12492 case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op1);
12493 case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op1);
12494 case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op1);
12495 case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op1);
12496 case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op1);
12497 case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op1);
12498 case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op1);
12499 default: return Register();
12500 }
12501}
12502
12503// FastEmit functions for ISD::ROTR.
12504
12505Register fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12506 if (RetVT.SimpleTy != MVT::i8)
12507 return Register();
12508 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12509 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12510 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL_ND, RC: &X86::GR8RegClass, Op0);
12511 }
12512 if ((!Subtarget->hasNDD())) {
12513 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12514 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL, RC: &X86::GR8RegClass, Op0);
12515 }
12516 return Register();
12517}
12518
12519Register fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12520 if (RetVT.SimpleTy != MVT::v4i32)
12521 return Register();
12522 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12523 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12524 }
12525 return Register();
12526}
12527
12528Register fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12529 if (RetVT.SimpleTy != MVT::v8i32)
12530 return Register();
12531 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12532 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12533 }
12534 return Register();
12535}
12536
12537Register fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12538 if (RetVT.SimpleTy != MVT::v16i32)
12539 return Register();
12540 if ((Subtarget->hasAVX512())) {
12541 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12542 }
12543 return Register();
12544}
12545
12546Register fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12547 if (RetVT.SimpleTy != MVT::v2i64)
12548 return Register();
12549 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12550 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12551 }
12552 return Register();
12553}
12554
12555Register fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12556 if (RetVT.SimpleTy != MVT::v4i64)
12557 return Register();
12558 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12559 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12560 }
12561 return Register();
12562}
12563
12564Register fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12565 if (RetVT.SimpleTy != MVT::v8i64)
12566 return Register();
12567 if ((Subtarget->hasAVX512())) {
12568 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12569 }
12570 return Register();
12571}
12572
12573Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12574 switch (VT.SimpleTy) {
12575 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op1);
12576 case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op1);
12577 case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op1);
12578 case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op1);
12579 case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op1);
12580 case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op1);
12581 case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op1);
12582 default: return Register();
12583 }
12584}
12585
12586// FastEmit functions for ISD::SADDSAT.
12587
12588Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12589 if (RetVT.SimpleTy != MVT::v16i8)
12590 return Register();
12591 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12592 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12593 }
12594 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12595 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12596 }
12597 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12598 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12599 }
12600 return Register();
12601}
12602
12603Register fastEmit_ISD_SADDSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12604 if (RetVT.SimpleTy != MVT::v32i8)
12605 return Register();
12606 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12607 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12608 }
12609 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12610 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12611 }
12612 return Register();
12613}
12614
12615Register fastEmit_ISD_SADDSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12616 if (RetVT.SimpleTy != MVT::v64i8)
12617 return Register();
12618 if ((Subtarget->hasBWI())) {
12619 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12620 }
12621 return Register();
12622}
12623
12624Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12625 if (RetVT.SimpleTy != MVT::v8i16)
12626 return Register();
12627 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12628 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12629 }
12630 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12631 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12632 }
12633 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12634 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12635 }
12636 return Register();
12637}
12638
12639Register fastEmit_ISD_SADDSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12640 if (RetVT.SimpleTy != MVT::v16i16)
12641 return Register();
12642 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12643 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12644 }
12645 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12646 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12647 }
12648 return Register();
12649}
12650
12651Register fastEmit_ISD_SADDSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12652 if (RetVT.SimpleTy != MVT::v32i16)
12653 return Register();
12654 if ((Subtarget->hasBWI())) {
12655 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12656 }
12657 return Register();
12658}
12659
12660Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12661 switch (VT.SimpleTy) {
12662 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12663 case MVT::v32i8: return fastEmit_ISD_SADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
12664 case MVT::v64i8: return fastEmit_ISD_SADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
12665 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12666 case MVT::v16i16: return fastEmit_ISD_SADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
12667 case MVT::v32i16: return fastEmit_ISD_SADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
12668 default: return Register();
12669 }
12670}
12671
12672// FastEmit functions for ISD::SHL.
12673
12674Register fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12675 if (RetVT.SimpleTy != MVT::i8)
12676 return Register();
12677 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12678 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12679 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL_ND, RC: &X86::GR8RegClass, Op0);
12680 }
12681 if ((!Subtarget->hasNDD())) {
12682 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12683 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL, RC: &X86::GR8RegClass, Op0);
12684 }
12685 return Register();
12686}
12687
12688Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12689 switch (VT.SimpleTy) {
12690 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op1);
12691 default: return Register();
12692 }
12693}
12694
12695// FastEmit functions for ISD::SMAX.
12696
12697Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12698 if (RetVT.SimpleTy != MVT::v16i8)
12699 return Register();
12700 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12701 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12702 }
12703 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12704 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12705 }
12706 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12707 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12708 }
12709 return Register();
12710}
12711
12712Register fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12713 if (RetVT.SimpleTy != MVT::v32i8)
12714 return Register();
12715 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12716 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12717 }
12718 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12719 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12720 }
12721 return Register();
12722}
12723
12724Register fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12725 if (RetVT.SimpleTy != MVT::v64i8)
12726 return Register();
12727 if ((Subtarget->hasBWI())) {
12728 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12729 }
12730 return Register();
12731}
12732
12733Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12734 if (RetVT.SimpleTy != MVT::v8i16)
12735 return Register();
12736 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12737 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12738 }
12739 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12740 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12741 }
12742 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12743 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12744 }
12745 return Register();
12746}
12747
12748Register fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12749 if (RetVT.SimpleTy != MVT::v16i16)
12750 return Register();
12751 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12752 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12753 }
12754 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12755 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12756 }
12757 return Register();
12758}
12759
12760Register fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12761 if (RetVT.SimpleTy != MVT::v32i16)
12762 return Register();
12763 if ((Subtarget->hasBWI())) {
12764 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12765 }
12766 return Register();
12767}
12768
12769Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12770 if (RetVT.SimpleTy != MVT::v4i32)
12771 return Register();
12772 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12773 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12774 }
12775 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12776 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12777 }
12778 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12779 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12780 }
12781 return Register();
12782}
12783
12784Register fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12785 if (RetVT.SimpleTy != MVT::v8i32)
12786 return Register();
12787 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12788 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12789 }
12790 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12791 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12792 }
12793 return Register();
12794}
12795
12796Register fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12797 if (RetVT.SimpleTy != MVT::v16i32)
12798 return Register();
12799 if ((Subtarget->hasAVX512())) {
12800 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12801 }
12802 return Register();
12803}
12804
12805Register fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12806 if (RetVT.SimpleTy != MVT::v2i64)
12807 return Register();
12808 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12809 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12810 }
12811 return Register();
12812}
12813
12814Register fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12815 if (RetVT.SimpleTy != MVT::v4i64)
12816 return Register();
12817 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12818 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12819 }
12820 return Register();
12821}
12822
12823Register fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12824 if (RetVT.SimpleTy != MVT::v8i64)
12825 return Register();
12826 if ((Subtarget->hasAVX512())) {
12827 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12828 }
12829 return Register();
12830}
12831
12832Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12833 switch (VT.SimpleTy) {
12834 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12835 case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
12836 case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
12837 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12838 case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
12839 case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
12840 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12841 case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
12842 case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
12843 case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
12844 case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
12845 case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
12846 default: return Register();
12847 }
12848}
12849
12850// FastEmit functions for ISD::SMIN.
12851
12852Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12853 if (RetVT.SimpleTy != MVT::v16i8)
12854 return Register();
12855 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12856 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12857 }
12858 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12859 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12860 }
12861 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12862 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12863 }
12864 return Register();
12865}
12866
12867Register fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12868 if (RetVT.SimpleTy != MVT::v32i8)
12869 return Register();
12870 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12871 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12872 }
12873 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12874 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12875 }
12876 return Register();
12877}
12878
12879Register fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12880 if (RetVT.SimpleTy != MVT::v64i8)
12881 return Register();
12882 if ((Subtarget->hasBWI())) {
12883 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12884 }
12885 return Register();
12886}
12887
12888Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12889 if (RetVT.SimpleTy != MVT::v8i16)
12890 return Register();
12891 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12892 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12893 }
12894 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12895 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12896 }
12897 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12898 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12899 }
12900 return Register();
12901}
12902
12903Register fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12904 if (RetVT.SimpleTy != MVT::v16i16)
12905 return Register();
12906 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12907 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12908 }
12909 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12910 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12911 }
12912 return Register();
12913}
12914
12915Register fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12916 if (RetVT.SimpleTy != MVT::v32i16)
12917 return Register();
12918 if ((Subtarget->hasBWI())) {
12919 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12920 }
12921 return Register();
12922}
12923
12924Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12925 if (RetVT.SimpleTy != MVT::v4i32)
12926 return Register();
12927 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12928 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12929 }
12930 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12931 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12932 }
12933 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12934 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12935 }
12936 return Register();
12937}
12938
12939Register fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12940 if (RetVT.SimpleTy != MVT::v8i32)
12941 return Register();
12942 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12943 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12944 }
12945 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12946 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12947 }
12948 return Register();
12949}
12950
12951Register fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12952 if (RetVT.SimpleTy != MVT::v16i32)
12953 return Register();
12954 if ((Subtarget->hasAVX512())) {
12955 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12956 }
12957 return Register();
12958}
12959
12960Register fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12961 if (RetVT.SimpleTy != MVT::v2i64)
12962 return Register();
12963 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12964 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12965 }
12966 return Register();
12967}
12968
12969Register fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12970 if (RetVT.SimpleTy != MVT::v4i64)
12971 return Register();
12972 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12973 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12974 }
12975 return Register();
12976}
12977
12978Register fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12979 if (RetVT.SimpleTy != MVT::v8i64)
12980 return Register();
12981 if ((Subtarget->hasAVX512())) {
12982 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12983 }
12984 return Register();
12985}
12986
12987Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12988 switch (VT.SimpleTy) {
12989 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12990 case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
12991 case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
12992 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12993 case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
12994 case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
12995 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12996 case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
12997 case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
12998 case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
12999 case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
13000 case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
13001 default: return Register();
13002 }
13003}
13004
13005// FastEmit functions for ISD::SRA.
13006
13007Register fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13008 if (RetVT.SimpleTy != MVT::i8)
13009 return Register();
13010 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
13011 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13012 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL_ND, RC: &X86::GR8RegClass, Op0);
13013 }
13014 if ((!Subtarget->hasNDD())) {
13015 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13016 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL, RC: &X86::GR8RegClass, Op0);
13017 }
13018 return Register();
13019}
13020
13021Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13022 switch (VT.SimpleTy) {
13023 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op1);
13024 default: return Register();
13025 }
13026}
13027
13028// FastEmit functions for ISD::SRL.
13029
13030Register fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13031 if (RetVT.SimpleTy != MVT::i8)
13032 return Register();
13033 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
13034 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13035 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL_ND, RC: &X86::GR8RegClass, Op0);
13036 }
13037 if ((!Subtarget->hasNDD())) {
13038 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13039 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL, RC: &X86::GR8RegClass, Op0);
13040 }
13041 return Register();
13042}
13043
13044Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13045 switch (VT.SimpleTy) {
13046 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op1);
13047 default: return Register();
13048 }
13049}
13050
13051// FastEmit functions for ISD::SSUBSAT.
13052
13053Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13054 if (RetVT.SimpleTy != MVT::v16i8)
13055 return Register();
13056 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13057 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13058 }
13059 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13060 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13061 }
13062 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13063 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13064 }
13065 return Register();
13066}
13067
13068Register fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13069 if (RetVT.SimpleTy != MVT::v32i8)
13070 return Register();
13071 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13072 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13073 }
13074 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13075 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13076 }
13077 return Register();
13078}
13079
13080Register fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13081 if (RetVT.SimpleTy != MVT::v64i8)
13082 return Register();
13083 if ((Subtarget->hasBWI())) {
13084 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13085 }
13086 return Register();
13087}
13088
13089Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13090 if (RetVT.SimpleTy != MVT::v8i16)
13091 return Register();
13092 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13093 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13094 }
13095 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13096 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13097 }
13098 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13099 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13100 }
13101 return Register();
13102}
13103
13104Register fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13105 if (RetVT.SimpleTy != MVT::v16i16)
13106 return Register();
13107 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13108 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13109 }
13110 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13111 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13112 }
13113 return Register();
13114}
13115
13116Register fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13117 if (RetVT.SimpleTy != MVT::v32i16)
13118 return Register();
13119 if ((Subtarget->hasBWI())) {
13120 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13121 }
13122 return Register();
13123}
13124
13125Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13126 switch (VT.SimpleTy) {
13127 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13128 case MVT::v32i8: return fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
13129 case MVT::v64i8: return fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
13130 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13131 case MVT::v16i16: return fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
13132 case MVT::v32i16: return fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
13133 default: return Register();
13134 }
13135}
13136
13137// FastEmit functions for ISD::STRICT_FADD.
13138
13139Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13140 if (RetVT.SimpleTy != MVT::f16)
13141 return Register();
13142 if ((Subtarget->hasFP16())) {
13143 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13144 }
13145 return Register();
13146}
13147
13148Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13149 if (RetVT.SimpleTy != MVT::f32)
13150 return Register();
13151 if ((Subtarget->hasAVX512())) {
13152 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13153 }
13154 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13155 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13156 }
13157 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13158 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13159 }
13160 if ((!Subtarget->hasSSE1())) {
13161 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13162 }
13163 return Register();
13164}
13165
13166Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13167 if (RetVT.SimpleTy != MVT::f64)
13168 return Register();
13169 if ((Subtarget->hasAVX512())) {
13170 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13171 }
13172 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13173 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13174 }
13175 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13176 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13177 }
13178 if ((!Subtarget->hasSSE2())) {
13179 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13180 }
13181 return Register();
13182}
13183
13184Register fastEmit_ISD_STRICT_FADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13185 if (RetVT.SimpleTy != MVT::f80)
13186 return Register();
13187 if ((Subtarget->hasX87())) {
13188 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13189 }
13190 return Register();
13191}
13192
13193Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13194 if (RetVT.SimpleTy != MVT::v8f16)
13195 return Register();
13196 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13197 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13198 }
13199 return Register();
13200}
13201
13202Register fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13203 if (RetVT.SimpleTy != MVT::v16f16)
13204 return Register();
13205 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13206 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13207 }
13208 return Register();
13209}
13210
13211Register fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13212 if (RetVT.SimpleTy != MVT::v32f16)
13213 return Register();
13214 if ((Subtarget->hasFP16())) {
13215 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13216 }
13217 return Register();
13218}
13219
13220Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13221 if (RetVT.SimpleTy != MVT::v4f32)
13222 return Register();
13223 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13224 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13225 }
13226 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13227 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13228 }
13229 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13230 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13231 }
13232 return Register();
13233}
13234
13235Register fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13236 if (RetVT.SimpleTy != MVT::v8f32)
13237 return Register();
13238 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13239 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13240 }
13241 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13242 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13243 }
13244 return Register();
13245}
13246
13247Register fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13248 if (RetVT.SimpleTy != MVT::v16f32)
13249 return Register();
13250 if ((Subtarget->hasAVX512())) {
13251 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13252 }
13253 return Register();
13254}
13255
13256Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13257 if (RetVT.SimpleTy != MVT::v2f64)
13258 return Register();
13259 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13260 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13261 }
13262 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13263 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13264 }
13265 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13266 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13267 }
13268 return Register();
13269}
13270
13271Register fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13272 if (RetVT.SimpleTy != MVT::v4f64)
13273 return Register();
13274 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13275 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13276 }
13277 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13278 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13279 }
13280 return Register();
13281}
13282
13283Register fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13284 if (RetVT.SimpleTy != MVT::v8f64)
13285 return Register();
13286 if ((Subtarget->hasAVX512())) {
13287 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13288 }
13289 return Register();
13290}
13291
13292Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13293 switch (VT.SimpleTy) {
13294 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
13295 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
13296 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
13297 case MVT::f80: return fastEmit_ISD_STRICT_FADD_MVT_f80_rr(RetVT, Op0, Op1);
13298 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
13299 case MVT::v16f16: return fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
13300 case MVT::v32f16: return fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
13301 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
13302 case MVT::v8f32: return fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
13303 case MVT::v16f32: return fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
13304 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
13305 case MVT::v4f64: return fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
13306 case MVT::v8f64: return fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
13307 default: return Register();
13308 }
13309}
13310
13311// FastEmit functions for ISD::STRICT_FDIV.
13312
13313Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13314 if (RetVT.SimpleTy != MVT::f16)
13315 return Register();
13316 if ((Subtarget->hasFP16())) {
13317 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13318 }
13319 return Register();
13320}
13321
13322Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13323 if (RetVT.SimpleTy != MVT::f32)
13324 return Register();
13325 if ((Subtarget->hasAVX512())) {
13326 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13327 }
13328 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13329 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13330 }
13331 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13332 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13333 }
13334 if ((!Subtarget->hasSSE1())) {
13335 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13336 }
13337 return Register();
13338}
13339
13340Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13341 if (RetVT.SimpleTy != MVT::f64)
13342 return Register();
13343 if ((Subtarget->hasAVX512())) {
13344 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13345 }
13346 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13347 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13348 }
13349 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13350 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13351 }
13352 if ((!Subtarget->hasSSE2())) {
13353 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13354 }
13355 return Register();
13356}
13357
13358Register fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13359 if (RetVT.SimpleTy != MVT::f80)
13360 return Register();
13361 if ((Subtarget->hasX87())) {
13362 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13363 }
13364 return Register();
13365}
13366
13367Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13368 if (RetVT.SimpleTy != MVT::v8f16)
13369 return Register();
13370 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13371 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13372 }
13373 return Register();
13374}
13375
13376Register fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13377 if (RetVT.SimpleTy != MVT::v16f16)
13378 return Register();
13379 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13380 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13381 }
13382 return Register();
13383}
13384
13385Register fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13386 if (RetVT.SimpleTy != MVT::v32f16)
13387 return Register();
13388 if ((Subtarget->hasFP16())) {
13389 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13390 }
13391 return Register();
13392}
13393
13394Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13395 if (RetVT.SimpleTy != MVT::v4f32)
13396 return Register();
13397 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13398 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13399 }
13400 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13401 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13402 }
13403 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13404 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13405 }
13406 return Register();
13407}
13408
13409Register fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13410 if (RetVT.SimpleTy != MVT::v8f32)
13411 return Register();
13412 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13413 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13414 }
13415 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13416 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13417 }
13418 return Register();
13419}
13420
13421Register fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13422 if (RetVT.SimpleTy != MVT::v16f32)
13423 return Register();
13424 if ((Subtarget->hasAVX512())) {
13425 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13426 }
13427 return Register();
13428}
13429
13430Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13431 if (RetVT.SimpleTy != MVT::v2f64)
13432 return Register();
13433 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13434 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13435 }
13436 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13437 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13438 }
13439 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13440 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13441 }
13442 return Register();
13443}
13444
13445Register fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13446 if (RetVT.SimpleTy != MVT::v4f64)
13447 return Register();
13448 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13449 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13450 }
13451 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13452 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13453 }
13454 return Register();
13455}
13456
13457Register fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13458 if (RetVT.SimpleTy != MVT::v8f64)
13459 return Register();
13460 if ((Subtarget->hasAVX512())) {
13461 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13462 }
13463 return Register();
13464}
13465
13466Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13467 switch (VT.SimpleTy) {
13468 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
13469 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
13470 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
13471 case MVT::f80: return fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
13472 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
13473 case MVT::v16f16: return fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
13474 case MVT::v32f16: return fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
13475 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
13476 case MVT::v8f32: return fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
13477 case MVT::v16f32: return fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
13478 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
13479 case MVT::v4f64: return fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
13480 case MVT::v8f64: return fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
13481 default: return Register();
13482 }
13483}
13484
13485// FastEmit functions for ISD::STRICT_FMUL.
13486
13487Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13488 if (RetVT.SimpleTy != MVT::f16)
13489 return Register();
13490 if ((Subtarget->hasFP16())) {
13491 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13492 }
13493 return Register();
13494}
13495
13496Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13497 if (RetVT.SimpleTy != MVT::f32)
13498 return Register();
13499 if ((Subtarget->hasAVX512())) {
13500 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13501 }
13502 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13503 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13504 }
13505 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13506 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13507 }
13508 if ((!Subtarget->hasSSE1())) {
13509 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13510 }
13511 return Register();
13512}
13513
13514Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13515 if (RetVT.SimpleTy != MVT::f64)
13516 return Register();
13517 if ((Subtarget->hasAVX512())) {
13518 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13519 }
13520 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13521 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13522 }
13523 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13524 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13525 }
13526 if ((!Subtarget->hasSSE2())) {
13527 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13528 }
13529 return Register();
13530}
13531
13532Register fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13533 if (RetVT.SimpleTy != MVT::f80)
13534 return Register();
13535 if ((Subtarget->hasX87())) {
13536 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13537 }
13538 return Register();
13539}
13540
13541Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13542 if (RetVT.SimpleTy != MVT::v8f16)
13543 return Register();
13544 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13545 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13546 }
13547 return Register();
13548}
13549
13550Register fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13551 if (RetVT.SimpleTy != MVT::v16f16)
13552 return Register();
13553 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13554 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13555 }
13556 return Register();
13557}
13558
13559Register fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13560 if (RetVT.SimpleTy != MVT::v32f16)
13561 return Register();
13562 if ((Subtarget->hasFP16())) {
13563 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13564 }
13565 return Register();
13566}
13567
13568Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13569 if (RetVT.SimpleTy != MVT::v4f32)
13570 return Register();
13571 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13572 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13573 }
13574 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13575 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13576 }
13577 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13578 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13579 }
13580 return Register();
13581}
13582
13583Register fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13584 if (RetVT.SimpleTy != MVT::v8f32)
13585 return Register();
13586 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13587 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13588 }
13589 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13590 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13591 }
13592 return Register();
13593}
13594
13595Register fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13596 if (RetVT.SimpleTy != MVT::v16f32)
13597 return Register();
13598 if ((Subtarget->hasAVX512())) {
13599 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13600 }
13601 return Register();
13602}
13603
13604Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13605 if (RetVT.SimpleTy != MVT::v2f64)
13606 return Register();
13607 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13608 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13609 }
13610 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13611 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13612 }
13613 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13614 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13615 }
13616 return Register();
13617}
13618
13619Register fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13620 if (RetVT.SimpleTy != MVT::v4f64)
13621 return Register();
13622 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13623 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13624 }
13625 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13626 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13627 }
13628 return Register();
13629}
13630
13631Register fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13632 if (RetVT.SimpleTy != MVT::v8f64)
13633 return Register();
13634 if ((Subtarget->hasAVX512())) {
13635 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13636 }
13637 return Register();
13638}
13639
13640Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13641 switch (VT.SimpleTy) {
13642 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13643 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13644 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13645 case MVT::f80: return fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
13646 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13647 case MVT::v16f16: return fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
13648 case MVT::v32f16: return fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
13649 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13650 case MVT::v8f32: return fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
13651 case MVT::v16f32: return fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
13652 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13653 case MVT::v4f64: return fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
13654 case MVT::v8f64: return fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
13655 default: return Register();
13656 }
13657}
13658
13659// FastEmit functions for ISD::STRICT_FSUB.
13660
13661Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13662 if (RetVT.SimpleTy != MVT::f16)
13663 return Register();
13664 if ((Subtarget->hasFP16())) {
13665 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13666 }
13667 return Register();
13668}
13669
13670Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13671 if (RetVT.SimpleTy != MVT::f32)
13672 return Register();
13673 if ((Subtarget->hasAVX512())) {
13674 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13675 }
13676 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13677 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13678 }
13679 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13680 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13681 }
13682 if ((!Subtarget->hasSSE1())) {
13683 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13684 }
13685 return Register();
13686}
13687
13688Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13689 if (RetVT.SimpleTy != MVT::f64)
13690 return Register();
13691 if ((Subtarget->hasAVX512())) {
13692 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13693 }
13694 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13695 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13696 }
13697 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13698 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13699 }
13700 if ((!Subtarget->hasSSE2())) {
13701 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13702 }
13703 return Register();
13704}
13705
13706Register fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13707 if (RetVT.SimpleTy != MVT::f80)
13708 return Register();
13709 if ((Subtarget->hasX87())) {
13710 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13711 }
13712 return Register();
13713}
13714
13715Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13716 if (RetVT.SimpleTy != MVT::v8f16)
13717 return Register();
13718 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13719 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13720 }
13721 return Register();
13722}
13723
13724Register fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13725 if (RetVT.SimpleTy != MVT::v16f16)
13726 return Register();
13727 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13728 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13729 }
13730 return Register();
13731}
13732
13733Register fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13734 if (RetVT.SimpleTy != MVT::v32f16)
13735 return Register();
13736 if ((Subtarget->hasFP16())) {
13737 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13738 }
13739 return Register();
13740}
13741
13742Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13743 if (RetVT.SimpleTy != MVT::v4f32)
13744 return Register();
13745 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13746 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13747 }
13748 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13749 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13750 }
13751 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13752 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13753 }
13754 return Register();
13755}
13756
13757Register fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13758 if (RetVT.SimpleTy != MVT::v8f32)
13759 return Register();
13760 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13761 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13762 }
13763 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13764 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13765 }
13766 return Register();
13767}
13768
13769Register fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13770 if (RetVT.SimpleTy != MVT::v16f32)
13771 return Register();
13772 if ((Subtarget->hasAVX512())) {
13773 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13774 }
13775 return Register();
13776}
13777
13778Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13779 if (RetVT.SimpleTy != MVT::v2f64)
13780 return Register();
13781 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13782 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13783 }
13784 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13785 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13786 }
13787 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13788 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13789 }
13790 return Register();
13791}
13792
13793Register fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13794 if (RetVT.SimpleTy != MVT::v4f64)
13795 return Register();
13796 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13797 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13798 }
13799 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13800 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13801 }
13802 return Register();
13803}
13804
13805Register fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13806 if (RetVT.SimpleTy != MVT::v8f64)
13807 return Register();
13808 if ((Subtarget->hasAVX512())) {
13809 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13810 }
13811 return Register();
13812}
13813
13814Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13815 switch (VT.SimpleTy) {
13816 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13817 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13818 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13819 case MVT::f80: return fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
13820 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13821 case MVT::v16f16: return fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
13822 case MVT::v32f16: return fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
13823 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13824 case MVT::v8f32: return fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
13825 case MVT::v16f32: return fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
13826 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13827 case MVT::v4f64: return fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
13828 case MVT::v8f64: return fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
13829 default: return Register();
13830 }
13831}
13832
13833// FastEmit functions for ISD::SUB.
13834
13835Register fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13836 if (RetVT.SimpleTy != MVT::i8)
13837 return Register();
13838 if ((Subtarget->hasNDD())) {
13839 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
13840 }
13841 if ((!Subtarget->hasNDD())) {
13842 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr, RC: &X86::GR8RegClass, Op0, Op1);
13843 }
13844 return Register();
13845}
13846
13847Register fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
13848 if (RetVT.SimpleTy != MVT::i16)
13849 return Register();
13850 if ((Subtarget->hasNDD())) {
13851 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
13852 }
13853 if ((!Subtarget->hasNDD())) {
13854 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr, RC: &X86::GR16RegClass, Op0, Op1);
13855 }
13856 return Register();
13857}
13858
13859Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13860 if (RetVT.SimpleTy != MVT::i32)
13861 return Register();
13862 if ((Subtarget->hasNDD())) {
13863 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
13864 }
13865 if ((!Subtarget->hasNDD())) {
13866 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr, RC: &X86::GR32RegClass, Op0, Op1);
13867 }
13868 return Register();
13869}
13870
13871Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13872 if (RetVT.SimpleTy != MVT::i64)
13873 return Register();
13874 if ((Subtarget->hasNDD())) {
13875 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
13876 }
13877 if ((!Subtarget->hasNDD())) {
13878 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr, RC: &X86::GR64RegClass, Op0, Op1);
13879 }
13880 return Register();
13881}
13882
13883Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13884 if (RetVT.SimpleTy != MVT::v16i8)
13885 return Register();
13886 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13887 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13888 }
13889 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13890 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
13891 }
13892 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13893 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
13894 }
13895 return Register();
13896}
13897
13898Register fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13899 if (RetVT.SimpleTy != MVT::v32i8)
13900 return Register();
13901 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13902 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13903 }
13904 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13905 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13906 }
13907 return Register();
13908}
13909
13910Register fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13911 if (RetVT.SimpleTy != MVT::v64i8)
13912 return Register();
13913 if ((Subtarget->hasBWI())) {
13914 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13915 }
13916 return Register();
13917}
13918
13919Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13920 if (RetVT.SimpleTy != MVT::v8i16)
13921 return Register();
13922 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13923 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13924 }
13925 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13926 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
13927 }
13928 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13929 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
13930 }
13931 return Register();
13932}
13933
13934Register fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13935 if (RetVT.SimpleTy != MVT::v16i16)
13936 return Register();
13937 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13938 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13939 }
13940 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13941 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13942 }
13943 return Register();
13944}
13945
13946Register fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13947 if (RetVT.SimpleTy != MVT::v32i16)
13948 return Register();
13949 if ((Subtarget->hasBWI())) {
13950 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13951 }
13952 return Register();
13953}
13954
13955Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13956 if (RetVT.SimpleTy != MVT::v4i32)
13957 return Register();
13958 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13959 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13960 }
13961 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13962 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
13963 }
13964 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13965 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
13966 }
13967 return Register();
13968}
13969
13970Register fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
13971 if (RetVT.SimpleTy != MVT::v8i32)
13972 return Register();
13973 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13974 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13975 }
13976 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13977 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13978 }
13979 return Register();
13980}
13981
13982Register fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
13983 if (RetVT.SimpleTy != MVT::v16i32)
13984 return Register();
13985 if ((Subtarget->hasAVX512())) {
13986 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13987 }
13988 return Register();
13989}
13990
13991Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13992 if (RetVT.SimpleTy != MVT::v2i64)
13993 return Register();
13994 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13995 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13996 }
13997 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13998 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
13999 }
14000 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14001 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
14002 }
14003 return Register();
14004}
14005
14006Register fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14007 if (RetVT.SimpleTy != MVT::v4i64)
14008 return Register();
14009 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14010 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14011 }
14012 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14013 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQYrr, RC: &X86::VR256RegClass, Op0, Op1);
14014 }
14015 return Register();
14016}
14017
14018Register fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14019 if (RetVT.SimpleTy != MVT::v8i64)
14020 return Register();
14021 if ((Subtarget->hasAVX512())) {
14022 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14023 }
14024 return Register();
14025}
14026
14027Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14028 switch (VT.SimpleTy) {
14029 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op1);
14030 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op1);
14031 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
14032 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
14033 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
14034 case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op1);
14035 case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op1);
14036 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
14037 case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op1);
14038 case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op1);
14039 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
14040 case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op1);
14041 case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op1);
14042 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
14043 case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op1);
14044 case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op1);
14045 default: return Register();
14046 }
14047}
14048
14049// FastEmit functions for ISD::UADDSAT.
14050
14051Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14052 if (RetVT.SimpleTy != MVT::v16i8)
14053 return Register();
14054 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14055 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14056 }
14057 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14058 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14059 }
14060 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14061 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14062 }
14063 return Register();
14064}
14065
14066Register fastEmit_ISD_UADDSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14067 if (RetVT.SimpleTy != MVT::v32i8)
14068 return Register();
14069 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14070 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14071 }
14072 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14073 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14074 }
14075 return Register();
14076}
14077
14078Register fastEmit_ISD_UADDSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14079 if (RetVT.SimpleTy != MVT::v64i8)
14080 return Register();
14081 if ((Subtarget->hasBWI())) {
14082 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14083 }
14084 return Register();
14085}
14086
14087Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14088 if (RetVT.SimpleTy != MVT::v8i16)
14089 return Register();
14090 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14091 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14092 }
14093 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14094 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14095 }
14096 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14097 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14098 }
14099 return Register();
14100}
14101
14102Register fastEmit_ISD_UADDSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14103 if (RetVT.SimpleTy != MVT::v16i16)
14104 return Register();
14105 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14106 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14107 }
14108 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14109 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14110 }
14111 return Register();
14112}
14113
14114Register fastEmit_ISD_UADDSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14115 if (RetVT.SimpleTy != MVT::v32i16)
14116 return Register();
14117 if ((Subtarget->hasBWI())) {
14118 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14119 }
14120 return Register();
14121}
14122
14123Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14124 switch (VT.SimpleTy) {
14125 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14126 case MVT::v32i8: return fastEmit_ISD_UADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
14127 case MVT::v64i8: return fastEmit_ISD_UADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
14128 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14129 case MVT::v16i16: return fastEmit_ISD_UADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
14130 case MVT::v32i16: return fastEmit_ISD_UADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
14131 default: return Register();
14132 }
14133}
14134
14135// FastEmit functions for ISD::UMAX.
14136
14137Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14138 if (RetVT.SimpleTy != MVT::v16i8)
14139 return Register();
14140 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14141 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14142 }
14143 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14144 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14145 }
14146 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14147 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14148 }
14149 return Register();
14150}
14151
14152Register fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14153 if (RetVT.SimpleTy != MVT::v32i8)
14154 return Register();
14155 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14156 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14157 }
14158 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14159 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14160 }
14161 return Register();
14162}
14163
14164Register fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14165 if (RetVT.SimpleTy != MVT::v64i8)
14166 return Register();
14167 if ((Subtarget->hasBWI())) {
14168 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14169 }
14170 return Register();
14171}
14172
14173Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14174 if (RetVT.SimpleTy != MVT::v8i16)
14175 return Register();
14176 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14177 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14178 }
14179 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14180 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14181 }
14182 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14183 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14184 }
14185 return Register();
14186}
14187
14188Register fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14189 if (RetVT.SimpleTy != MVT::v16i16)
14190 return Register();
14191 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14192 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14193 }
14194 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14195 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14196 }
14197 return Register();
14198}
14199
14200Register fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14201 if (RetVT.SimpleTy != MVT::v32i16)
14202 return Register();
14203 if ((Subtarget->hasBWI())) {
14204 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14205 }
14206 return Register();
14207}
14208
14209Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14210 if (RetVT.SimpleTy != MVT::v4i32)
14211 return Register();
14212 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14213 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14214 }
14215 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14216 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14217 }
14218 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14219 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14220 }
14221 return Register();
14222}
14223
14224Register fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14225 if (RetVT.SimpleTy != MVT::v8i32)
14226 return Register();
14227 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14228 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14229 }
14230 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14231 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14232 }
14233 return Register();
14234}
14235
14236Register fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14237 if (RetVT.SimpleTy != MVT::v16i32)
14238 return Register();
14239 if ((Subtarget->hasAVX512())) {
14240 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14241 }
14242 return Register();
14243}
14244
14245Register fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14246 if (RetVT.SimpleTy != MVT::v2i64)
14247 return Register();
14248 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14249 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14250 }
14251 return Register();
14252}
14253
14254Register fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14255 if (RetVT.SimpleTy != MVT::v4i64)
14256 return Register();
14257 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14258 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14259 }
14260 return Register();
14261}
14262
14263Register fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14264 if (RetVT.SimpleTy != MVT::v8i64)
14265 return Register();
14266 if ((Subtarget->hasAVX512())) {
14267 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14268 }
14269 return Register();
14270}
14271
14272Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14273 switch (VT.SimpleTy) {
14274 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
14275 case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
14276 case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
14277 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
14278 case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
14279 case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
14280 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
14281 case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
14282 case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
14283 case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
14284 case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
14285 case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
14286 default: return Register();
14287 }
14288}
14289
14290// FastEmit functions for ISD::UMIN.
14291
14292Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14293 if (RetVT.SimpleTy != MVT::v16i8)
14294 return Register();
14295 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14296 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14297 }
14298 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14299 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14300 }
14301 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14302 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14303 }
14304 return Register();
14305}
14306
14307Register fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14308 if (RetVT.SimpleTy != MVT::v32i8)
14309 return Register();
14310 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14311 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14312 }
14313 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14314 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14315 }
14316 return Register();
14317}
14318
14319Register fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14320 if (RetVT.SimpleTy != MVT::v64i8)
14321 return Register();
14322 if ((Subtarget->hasBWI())) {
14323 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14324 }
14325 return Register();
14326}
14327
14328Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14329 if (RetVT.SimpleTy != MVT::v8i16)
14330 return Register();
14331 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14332 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14333 }
14334 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14335 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14336 }
14337 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14338 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14339 }
14340 return Register();
14341}
14342
14343Register fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14344 if (RetVT.SimpleTy != MVT::v16i16)
14345 return Register();
14346 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14347 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14348 }
14349 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14350 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14351 }
14352 return Register();
14353}
14354
14355Register fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14356 if (RetVT.SimpleTy != MVT::v32i16)
14357 return Register();
14358 if ((Subtarget->hasBWI())) {
14359 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14360 }
14361 return Register();
14362}
14363
14364Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14365 if (RetVT.SimpleTy != MVT::v4i32)
14366 return Register();
14367 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14368 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14369 }
14370 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14371 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14372 }
14373 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14374 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14375 }
14376 return Register();
14377}
14378
14379Register fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14380 if (RetVT.SimpleTy != MVT::v8i32)
14381 return Register();
14382 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14383 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14384 }
14385 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14386 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14387 }
14388 return Register();
14389}
14390
14391Register fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14392 if (RetVT.SimpleTy != MVT::v16i32)
14393 return Register();
14394 if ((Subtarget->hasAVX512())) {
14395 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14396 }
14397 return Register();
14398}
14399
14400Register fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14401 if (RetVT.SimpleTy != MVT::v2i64)
14402 return Register();
14403 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14404 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14405 }
14406 return Register();
14407}
14408
14409Register fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14410 if (RetVT.SimpleTy != MVT::v4i64)
14411 return Register();
14412 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14413 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14414 }
14415 return Register();
14416}
14417
14418Register fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14419 if (RetVT.SimpleTy != MVT::v8i64)
14420 return Register();
14421 if ((Subtarget->hasAVX512())) {
14422 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14423 }
14424 return Register();
14425}
14426
14427Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14428 switch (VT.SimpleTy) {
14429 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
14430 case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
14431 case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
14432 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
14433 case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
14434 case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
14435 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
14436 case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
14437 case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
14438 case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
14439 case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
14440 case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
14441 default: return Register();
14442 }
14443}
14444
14445// FastEmit functions for ISD::USUBSAT.
14446
14447Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14448 if (RetVT.SimpleTy != MVT::v16i8)
14449 return Register();
14450 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14451 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14452 }
14453 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14454 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14455 }
14456 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14457 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14458 }
14459 return Register();
14460}
14461
14462Register fastEmit_ISD_USUBSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14463 if (RetVT.SimpleTy != MVT::v32i8)
14464 return Register();
14465 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14466 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14467 }
14468 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14469 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14470 }
14471 return Register();
14472}
14473
14474Register fastEmit_ISD_USUBSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14475 if (RetVT.SimpleTy != MVT::v64i8)
14476 return Register();
14477 if ((Subtarget->hasBWI())) {
14478 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14479 }
14480 return Register();
14481}
14482
14483Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14484 if (RetVT.SimpleTy != MVT::v8i16)
14485 return Register();
14486 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14487 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14488 }
14489 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14490 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14491 }
14492 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14493 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14494 }
14495 return Register();
14496}
14497
14498Register fastEmit_ISD_USUBSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14499 if (RetVT.SimpleTy != MVT::v16i16)
14500 return Register();
14501 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14502 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14503 }
14504 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14505 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14506 }
14507 return Register();
14508}
14509
14510Register fastEmit_ISD_USUBSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14511 if (RetVT.SimpleTy != MVT::v32i16)
14512 return Register();
14513 if ((Subtarget->hasBWI())) {
14514 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14515 }
14516 return Register();
14517}
14518
14519Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14520 switch (VT.SimpleTy) {
14521 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14522 case MVT::v32i8: return fastEmit_ISD_USUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
14523 case MVT::v64i8: return fastEmit_ISD_USUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
14524 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14525 case MVT::v16i16: return fastEmit_ISD_USUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
14526 case MVT::v32i16: return fastEmit_ISD_USUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
14527 default: return Register();
14528 }
14529}
14530
14531// FastEmit functions for ISD::XOR.
14532
14533Register fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
14534 if (RetVT.SimpleTy != MVT::i8)
14535 return Register();
14536 if ((Subtarget->hasNDD())) {
14537 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
14538 }
14539 if ((!Subtarget->hasNDD())) {
14540 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr, RC: &X86::GR8RegClass, Op0, Op1);
14541 }
14542 return Register();
14543}
14544
14545Register fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
14546 if (RetVT.SimpleTy != MVT::i16)
14547 return Register();
14548 if ((Subtarget->hasNDD())) {
14549 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
14550 }
14551 if ((!Subtarget->hasNDD())) {
14552 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr, RC: &X86::GR16RegClass, Op0, Op1);
14553 }
14554 return Register();
14555}
14556
14557Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14558 if (RetVT.SimpleTy != MVT::i32)
14559 return Register();
14560 if ((Subtarget->hasNDD())) {
14561 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
14562 }
14563 if ((!Subtarget->hasNDD())) {
14564 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr, RC: &X86::GR32RegClass, Op0, Op1);
14565 }
14566 return Register();
14567}
14568
14569Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14570 if (RetVT.SimpleTy != MVT::i64)
14571 return Register();
14572 if ((Subtarget->hasNDD())) {
14573 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
14574 }
14575 if ((!Subtarget->hasNDD())) {
14576 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr, RC: &X86::GR64RegClass, Op0, Op1);
14577 }
14578 return Register();
14579}
14580
14581Register fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
14582 if (RetVT.SimpleTy != MVT::v8i1)
14583 return Register();
14584 if ((Subtarget->hasDQI())) {
14585 return fastEmitInst_rr(MachineInstOpcode: X86::KXORBkk, RC: &X86::VK8RegClass, Op0, Op1);
14586 }
14587 return Register();
14588}
14589
14590Register fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
14591 if (RetVT.SimpleTy != MVT::v16i1)
14592 return Register();
14593 if ((Subtarget->hasAVX512())) {
14594 return fastEmitInst_rr(MachineInstOpcode: X86::KXORWkk, RC: &X86::VK16RegClass, Op0, Op1);
14595 }
14596 return Register();
14597}
14598
14599Register fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
14600 if (RetVT.SimpleTy != MVT::v32i1)
14601 return Register();
14602 if ((Subtarget->hasBWI())) {
14603 return fastEmitInst_rr(MachineInstOpcode: X86::KXORDkk, RC: &X86::VK32RegClass, Op0, Op1);
14604 }
14605 return Register();
14606}
14607
14608Register fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
14609 if (RetVT.SimpleTy != MVT::v64i1)
14610 return Register();
14611 if ((Subtarget->hasBWI())) {
14612 return fastEmitInst_rr(MachineInstOpcode: X86::KXORQkk, RC: &X86::VK64RegClass, Op0, Op1);
14613 }
14614 return Register();
14615}
14616
14617Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14618 if (RetVT.SimpleTy != MVT::v16i8)
14619 return Register();
14620 if ((Subtarget->hasVLX())) {
14621 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14622 }
14623 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14624 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14625 }
14626 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14627 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14628 }
14629 return Register();
14630}
14631
14632Register fastEmit_ISD_XOR_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14633 if (RetVT.SimpleTy != MVT::v32i8)
14634 return Register();
14635 if ((Subtarget->hasVLX())) {
14636 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14637 }
14638 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14639 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14640 }
14641 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14642 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14643 }
14644 return Register();
14645}
14646
14647Register fastEmit_ISD_XOR_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14648 if (RetVT.SimpleTy != MVT::v64i8)
14649 return Register();
14650 if ((Subtarget->hasAVX512())) {
14651 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14652 }
14653 return Register();
14654}
14655
14656Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14657 if (RetVT.SimpleTy != MVT::v8i16)
14658 return Register();
14659 if ((Subtarget->hasVLX())) {
14660 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14661 }
14662 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14663 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14664 }
14665 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14666 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14667 }
14668 return Register();
14669}
14670
14671Register fastEmit_ISD_XOR_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14672 if (RetVT.SimpleTy != MVT::v16i16)
14673 return Register();
14674 if ((Subtarget->hasVLX())) {
14675 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14676 }
14677 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14678 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14679 }
14680 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14681 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14682 }
14683 return Register();
14684}
14685
14686Register fastEmit_ISD_XOR_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14687 if (RetVT.SimpleTy != MVT::v32i16)
14688 return Register();
14689 if ((Subtarget->hasAVX512())) {
14690 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14691 }
14692 return Register();
14693}
14694
14695Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14696 if (RetVT.SimpleTy != MVT::v4i32)
14697 return Register();
14698 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14699 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14700 }
14701 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14702 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14703 }
14704 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14705 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14706 }
14707 return Register();
14708}
14709
14710Register fastEmit_ISD_XOR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14711 if (RetVT.SimpleTy != MVT::v8i32)
14712 return Register();
14713 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14714 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14715 }
14716 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14717 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14718 }
14719 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14720 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14721 }
14722 return Register();
14723}
14724
14725Register fastEmit_ISD_XOR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14726 if (RetVT.SimpleTy != MVT::v16i32)
14727 return Register();
14728 if ((Subtarget->hasAVX512())) {
14729 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14730 }
14731 return Register();
14732}
14733
14734Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14735 if (RetVT.SimpleTy != MVT::v2i64)
14736 return Register();
14737 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14738 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14739 }
14740 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14741 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14742 }
14743 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14744 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14745 }
14746 return Register();
14747}
14748
14749Register fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14750 if (RetVT.SimpleTy != MVT::v4i64)
14751 return Register();
14752 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14753 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14754 }
14755 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14756 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14757 }
14758 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14759 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14760 }
14761 return Register();
14762}
14763
14764Register fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14765 if (RetVT.SimpleTy != MVT::v8i64)
14766 return Register();
14767 if ((Subtarget->hasAVX512())) {
14768 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14769 }
14770 return Register();
14771}
14772
14773Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14774 switch (VT.SimpleTy) {
14775 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op1);
14776 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op1);
14777 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14778 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14779 case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op1);
14780 case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op1);
14781 case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op1);
14782 case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op1);
14783 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14784 case MVT::v32i8: return fastEmit_ISD_XOR_MVT_v32i8_rr(RetVT, Op0, Op1);
14785 case MVT::v64i8: return fastEmit_ISD_XOR_MVT_v64i8_rr(RetVT, Op0, Op1);
14786 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14787 case MVT::v16i16: return fastEmit_ISD_XOR_MVT_v16i16_rr(RetVT, Op0, Op1);
14788 case MVT::v32i16: return fastEmit_ISD_XOR_MVT_v32i16_rr(RetVT, Op0, Op1);
14789 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14790 case MVT::v8i32: return fastEmit_ISD_XOR_MVT_v8i32_rr(RetVT, Op0, Op1);
14791 case MVT::v16i32: return fastEmit_ISD_XOR_MVT_v16i32_rr(RetVT, Op0, Op1);
14792 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14793 case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op1);
14794 case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op1);
14795 default: return Register();
14796 }
14797}
14798
14799// FastEmit functions for X86ISD::ADDSUB.
14800
14801Register fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
14802 if (RetVT.SimpleTy != MVT::v4f32)
14803 return Register();
14804 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
14805 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14806 }
14807 if ((Subtarget->hasAVX())) {
14808 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14809 }
14810 return Register();
14811}
14812
14813Register fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
14814 if (RetVT.SimpleTy != MVT::v8f32)
14815 return Register();
14816 if ((Subtarget->hasAVX())) {
14817 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14818 }
14819 return Register();
14820}
14821
14822Register fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14823 if (RetVT.SimpleTy != MVT::v2f64)
14824 return Register();
14825 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
14826 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14827 }
14828 if ((Subtarget->hasAVX())) {
14829 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14830 }
14831 return Register();
14832}
14833
14834Register fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14835 if (RetVT.SimpleTy != MVT::v4f64)
14836 return Register();
14837 if ((Subtarget->hasAVX())) {
14838 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14839 }
14840 return Register();
14841}
14842
14843Register fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14844 switch (VT.SimpleTy) {
14845 case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
14846 case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
14847 case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
14848 case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
14849 default: return Register();
14850 }
14851}
14852
14853// FastEmit functions for X86ISD::ANDNP.
14854
14855Register fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14856 if (RetVT.SimpleTy != MVT::v16i8)
14857 return Register();
14858 if ((Subtarget->hasVLX())) {
14859 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14860 }
14861 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14862 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14863 }
14864 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14865 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14866 }
14867 return Register();
14868}
14869
14870Register fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14871 if (RetVT.SimpleTy != MVT::v32i8)
14872 return Register();
14873 if ((Subtarget->hasVLX())) {
14874 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14875 }
14876 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14877 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14878 }
14879 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14880 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14881 }
14882 return Register();
14883}
14884
14885Register fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14886 if (RetVT.SimpleTy != MVT::v64i8)
14887 return Register();
14888 if ((Subtarget->hasAVX512())) {
14889 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14890 }
14891 return Register();
14892}
14893
14894Register fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14895 if (RetVT.SimpleTy != MVT::v8i16)
14896 return Register();
14897 if ((Subtarget->hasVLX())) {
14898 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14899 }
14900 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14901 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14902 }
14903 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14904 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14905 }
14906 return Register();
14907}
14908
14909Register fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14910 if (RetVT.SimpleTy != MVT::v16i16)
14911 return Register();
14912 if ((Subtarget->hasVLX())) {
14913 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14914 }
14915 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14916 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14917 }
14918 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14919 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14920 }
14921 return Register();
14922}
14923
14924Register fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14925 if (RetVT.SimpleTy != MVT::v32i16)
14926 return Register();
14927 if ((Subtarget->hasAVX512())) {
14928 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14929 }
14930 return Register();
14931}
14932
14933Register fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14934 if (RetVT.SimpleTy != MVT::v4i32)
14935 return Register();
14936 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14937 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14938 }
14939 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14940 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14941 }
14942 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14943 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14944 }
14945 return Register();
14946}
14947
14948Register fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14949 if (RetVT.SimpleTy != MVT::v8i32)
14950 return Register();
14951 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14952 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14953 }
14954 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14955 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14956 }
14957 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14958 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14959 }
14960 return Register();
14961}
14962
14963Register fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14964 if (RetVT.SimpleTy != MVT::v16i32)
14965 return Register();
14966 if ((Subtarget->hasAVX512())) {
14967 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14968 }
14969 return Register();
14970}
14971
14972Register fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14973 if (RetVT.SimpleTy != MVT::v2i64)
14974 return Register();
14975 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14976 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14977 }
14978 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14979 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14980 }
14981 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14982 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14983 }
14984 return Register();
14985}
14986
14987Register fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14988 if (RetVT.SimpleTy != MVT::v4i64)
14989 return Register();
14990 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14991 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14992 }
14993 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14994 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14995 }
14996 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14997 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14998 }
14999 return Register();
15000}
15001
15002Register fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
15003 if (RetVT.SimpleTy != MVT::v8i64)
15004 return Register();
15005 if ((Subtarget->hasAVX512())) {
15006 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15007 }
15008 return Register();
15009}
15010
15011Register fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15012 switch (VT.SimpleTy) {
15013 case MVT::v16i8: return fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(RetVT, Op0, Op1);
15014 case MVT::v32i8: return fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(RetVT, Op0, Op1);
15015 case MVT::v64i8: return fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(RetVT, Op0, Op1);
15016 case MVT::v8i16: return fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(RetVT, Op0, Op1);
15017 case MVT::v16i16: return fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(RetVT, Op0, Op1);
15018 case MVT::v32i16: return fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(RetVT, Op0, Op1);
15019 case MVT::v4i32: return fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(RetVT, Op0, Op1);
15020 case MVT::v8i32: return fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(RetVT, Op0, Op1);
15021 case MVT::v16i32: return fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(RetVT, Op0, Op1);
15022 case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op1);
15023 case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op1);
15024 case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op1);
15025 default: return Register();
15026 }
15027}
15028
15029// FastEmit functions for X86ISD::BEXTR.
15030
15031Register fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15032 if (RetVT.SimpleTy != MVT::i32)
15033 return Register();
15034 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15035 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
15036 }
15037 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
15038 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr, RC: &X86::GR32RegClass, Op0, Op1);
15039 }
15040 return Register();
15041}
15042
15043Register fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15044 if (RetVT.SimpleTy != MVT::i64)
15045 return Register();
15046 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15047 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
15048 }
15049 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
15050 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr, RC: &X86::GR64RegClass, Op0, Op1);
15051 }
15052 return Register();
15053}
15054
15055Register fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15056 switch (VT.SimpleTy) {
15057 case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op1);
15058 case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op1);
15059 default: return Register();
15060 }
15061}
15062
15063// FastEmit functions for X86ISD::BT.
15064
15065Register fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
15066 if (RetVT.SimpleTy != MVT::i32)
15067 return Register();
15068 return fastEmitInst_rr(MachineInstOpcode: X86::BT16rr, RC: &X86::GR16RegClass, Op0, Op1);
15069}
15070
15071Register fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15072 if (RetVT.SimpleTy != MVT::i32)
15073 return Register();
15074 return fastEmitInst_rr(MachineInstOpcode: X86::BT32rr, RC: &X86::GR32RegClass, Op0, Op1);
15075}
15076
15077Register fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15078 if (RetVT.SimpleTy != MVT::i32)
15079 return Register();
15080 return fastEmitInst_rr(MachineInstOpcode: X86::BT64rr, RC: &X86::GR64RegClass, Op0, Op1);
15081}
15082
15083Register fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15084 switch (VT.SimpleTy) {
15085 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op1);
15086 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op1);
15087 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op1);
15088 default: return Register();
15089 }
15090}
15091
15092// FastEmit functions for X86ISD::BZHI.
15093
15094Register fastEmit_X86ISD_BZHI_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15095 if (RetVT.SimpleTy != MVT::i32)
15096 return Register();
15097 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15098 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
15099 }
15100 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15101 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr, RC: &X86::GR32RegClass, Op0, Op1);
15102 }
15103 return Register();
15104}
15105
15106Register fastEmit_X86ISD_BZHI_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15107 if (RetVT.SimpleTy != MVT::i64)
15108 return Register();
15109 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15110 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
15111 }
15112 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15113 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr, RC: &X86::GR64RegClass, Op0, Op1);
15114 }
15115 return Register();
15116}
15117
15118Register fastEmit_X86ISD_BZHI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15119 switch (VT.SimpleTy) {
15120 case MVT::i32: return fastEmit_X86ISD_BZHI_MVT_i32_rr(RetVT, Op0, Op1);
15121 case MVT::i64: return fastEmit_X86ISD_BZHI_MVT_i64_rr(RetVT, Op0, Op1);
15122 default: return Register();
15123 }
15124}
15125
15126// FastEmit functions for X86ISD::CMP.
15127
15128Register fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
15129 if (RetVT.SimpleTy != MVT::i32)
15130 return Register();
15131 return fastEmitInst_rr(MachineInstOpcode: X86::CMP8rr, RC: &X86::GR8RegClass, Op0, Op1);
15132}
15133
15134Register fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
15135 if (RetVT.SimpleTy != MVT::i32)
15136 return Register();
15137 return fastEmitInst_rr(MachineInstOpcode: X86::CMP16rr, RC: &X86::GR16RegClass, Op0, Op1);
15138}
15139
15140Register fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15141 if (RetVT.SimpleTy != MVT::i32)
15142 return Register();
15143 return fastEmitInst_rr(MachineInstOpcode: X86::CMP32rr, RC: &X86::GR32RegClass, Op0, Op1);
15144}
15145
15146Register fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15147 if (RetVT.SimpleTy != MVT::i32)
15148 return Register();
15149 return fastEmitInst_rr(MachineInstOpcode: X86::CMP64rr, RC: &X86::GR64RegClass, Op0, Op1);
15150}
15151
15152Register fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15153 switch (VT.SimpleTy) {
15154 case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op1);
15155 case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op1);
15156 case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
15157 case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op1);
15158 default: return Register();
15159 }
15160}
15161
15162// FastEmit functions for X86ISD::COMI.
15163
15164Register fastEmit_X86ISD_COMI_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15165 if (RetVT.SimpleTy != MVT::i32)
15166 return Register();
15167 if ((Subtarget->hasFP16())) {
15168 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15169 }
15170 return Register();
15171}
15172
15173Register fastEmit_X86ISD_COMI_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
15174 if (RetVT.SimpleTy != MVT::i32)
15175 return Register();
15176 if ((Subtarget->hasAVX10_2())) {
15177 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISBF16Zrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15178 }
15179 return Register();
15180}
15181
15182Register fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15183 if (RetVT.SimpleTy != MVT::i32)
15184 return Register();
15185 if ((Subtarget->hasAVX512())) {
15186 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15187 }
15188 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15189 return fastEmitInst_rr(MachineInstOpcode: X86::COMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15190 }
15191 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15192 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15193 }
15194 return Register();
15195}
15196
15197Register fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15198 if (RetVT.SimpleTy != MVT::i32)
15199 return Register();
15200 if ((Subtarget->hasAVX512())) {
15201 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15202 }
15203 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15204 return fastEmitInst_rr(MachineInstOpcode: X86::COMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15205 }
15206 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15207 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15208 }
15209 return Register();
15210}
15211
15212Register fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15213 switch (VT.SimpleTy) {
15214 case MVT::v8f16: return fastEmit_X86ISD_COMI_MVT_v8f16_rr(RetVT, Op0, Op1);
15215 case MVT::v8bf16: return fastEmit_X86ISD_COMI_MVT_v8bf16_rr(RetVT, Op0, Op1);
15216 case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op1);
15217 case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op1);
15218 default: return Register();
15219 }
15220}
15221
15222// FastEmit functions for X86ISD::COMX.
15223
15224Register fastEmit_X86ISD_COMX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15225 if (RetVT.SimpleTy != MVT::i32)
15226 return Register();
15227 if ((Subtarget->hasAVX10_2())) {
15228 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15229 }
15230 return Register();
15231}
15232
15233Register fastEmit_X86ISD_COMX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15234 if (RetVT.SimpleTy != MVT::i32)
15235 return Register();
15236 if ((Subtarget->hasAVX10_2())) {
15237 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15238 }
15239 return Register();
15240}
15241
15242Register fastEmit_X86ISD_COMX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15243 if (RetVT.SimpleTy != MVT::i32)
15244 return Register();
15245 if ((Subtarget->hasAVX10_2())) {
15246 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15247 }
15248 return Register();
15249}
15250
15251Register fastEmit_X86ISD_COMX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15252 switch (VT.SimpleTy) {
15253 case MVT::v8f16: return fastEmit_X86ISD_COMX_MVT_v8f16_rr(RetVT, Op0, Op1);
15254 case MVT::v4f32: return fastEmit_X86ISD_COMX_MVT_v4f32_rr(RetVT, Op0, Op1);
15255 case MVT::v2f64: return fastEmit_X86ISD_COMX_MVT_v2f64_rr(RetVT, Op0, Op1);
15256 default: return Register();
15257 }
15258}
15259
15260// FastEmit functions for X86ISD::FADDS.
15261
15262Register fastEmit_X86ISD_FADDS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15263 if (RetVT.SimpleTy != MVT::v8f16)
15264 return Register();
15265 if ((Subtarget->hasFP16())) {
15266 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15267 }
15268 return Register();
15269}
15270
15271Register fastEmit_X86ISD_FADDS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15272 if (RetVT.SimpleTy != MVT::v4f32)
15273 return Register();
15274 if ((Subtarget->hasAVX512())) {
15275 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15276 }
15277 return Register();
15278}
15279
15280Register fastEmit_X86ISD_FADDS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15281 if (RetVT.SimpleTy != MVT::v2f64)
15282 return Register();
15283 if ((Subtarget->hasAVX512())) {
15284 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15285 }
15286 return Register();
15287}
15288
15289Register fastEmit_X86ISD_FADDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15290 switch (VT.SimpleTy) {
15291 case MVT::v8f16: return fastEmit_X86ISD_FADDS_MVT_v8f16_rr(RetVT, Op0, Op1);
15292 case MVT::v4f32: return fastEmit_X86ISD_FADDS_MVT_v4f32_rr(RetVT, Op0, Op1);
15293 case MVT::v2f64: return fastEmit_X86ISD_FADDS_MVT_v2f64_rr(RetVT, Op0, Op1);
15294 default: return Register();
15295 }
15296}
15297
15298// FastEmit functions for X86ISD::FAND.
15299
15300Register fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
15301 if (RetVT.SimpleTy != MVT::f128)
15302 return Register();
15303 if ((Subtarget->hasVLX())) {
15304 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15305 }
15306 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15307 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15308 }
15309 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15310 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15311 }
15312 return Register();
15313}
15314
15315Register fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15316 if (RetVT.SimpleTy != MVT::v4f32)
15317 return Register();
15318 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15319}
15320
15321Register fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15322 switch (VT.SimpleTy) {
15323 case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op1);
15324 case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op1);
15325 default: return Register();
15326 }
15327}
15328
15329// FastEmit functions for X86ISD::FANDN.
15330
15331Register fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15332 if (RetVT.SimpleTy != MVT::v4f32)
15333 return Register();
15334 return fastEmitInst_rr(MachineInstOpcode: X86::ANDNPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15335}
15336
15337Register fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15338 switch (VT.SimpleTy) {
15339 case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op1);
15340 default: return Register();
15341 }
15342}
15343
15344// FastEmit functions for X86ISD::FCMP.
15345
15346Register fastEmit_X86ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
15347 if (RetVT.SimpleTy != MVT::i32)
15348 return Register();
15349 if ((Subtarget->hasFP16())) {
15350 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
15351 }
15352 return Register();
15353}
15354
15355Register fastEmit_X86ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
15356 if (RetVT.SimpleTy != MVT::i32)
15357 return Register();
15358 if ((Subtarget->hasAVX512())) {
15359 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
15360 }
15361 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15362 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
15363 }
15364 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15365 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
15366 }
15367 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
15368 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
15369 }
15370 return Register();
15371}
15372
15373Register fastEmit_X86ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
15374 if (RetVT.SimpleTy != MVT::i32)
15375 return Register();
15376 if ((Subtarget->hasAVX512())) {
15377 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
15378 }
15379 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15380 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
15381 }
15382 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15383 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
15384 }
15385 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
15386 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
15387 }
15388 return Register();
15389}
15390
15391Register fastEmit_X86ISD_FCMP_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
15392 if (RetVT.SimpleTy != MVT::i32)
15393 return Register();
15394 if ((Subtarget->canUseCMOV())) {
15395 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
15396 }
15397 return Register();
15398}
15399
15400Register fastEmit_X86ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15401 switch (VT.SimpleTy) {
15402 case MVT::f16: return fastEmit_X86ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
15403 case MVT::f32: return fastEmit_X86ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
15404 case MVT::f64: return fastEmit_X86ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
15405 case MVT::f80: return fastEmit_X86ISD_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
15406 default: return Register();
15407 }
15408}
15409
15410// FastEmit functions for X86ISD::FDIVS.
15411
15412Register fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15413 if (RetVT.SimpleTy != MVT::v8f16)
15414 return Register();
15415 if ((Subtarget->hasFP16())) {
15416 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15417 }
15418 return Register();
15419}
15420
15421Register fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15422 if (RetVT.SimpleTy != MVT::v4f32)
15423 return Register();
15424 if ((Subtarget->hasAVX512())) {
15425 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15426 }
15427 return Register();
15428}
15429
15430Register fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15431 if (RetVT.SimpleTy != MVT::v2f64)
15432 return Register();
15433 if ((Subtarget->hasAVX512())) {
15434 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15435 }
15436 return Register();
15437}
15438
15439Register fastEmit_X86ISD_FDIVS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15440 switch (VT.SimpleTy) {
15441 case MVT::v8f16: return fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(RetVT, Op0, Op1);
15442 case MVT::v4f32: return fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(RetVT, Op0, Op1);
15443 case MVT::v2f64: return fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(RetVT, Op0, Op1);
15444 default: return Register();
15445 }
15446}
15447
15448// FastEmit functions for X86ISD::FGETEXPS.
15449
15450Register fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15451 if (RetVT.SimpleTy != MVT::v8f16)
15452 return Register();
15453 if ((Subtarget->hasFP16())) {
15454 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZr, RC: &X86::VR128XRegClass, Op0, Op1);
15455 }
15456 return Register();
15457}
15458
15459Register fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15460 if (RetVT.SimpleTy != MVT::v4f32)
15461 return Register();
15462 if ((Subtarget->hasAVX512())) {
15463 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZr, RC: &X86::VR128XRegClass, Op0, Op1);
15464 }
15465 return Register();
15466}
15467
15468Register fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15469 if (RetVT.SimpleTy != MVT::v2f64)
15470 return Register();
15471 if ((Subtarget->hasAVX512())) {
15472 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZr, RC: &X86::VR128XRegClass, Op0, Op1);
15473 }
15474 return Register();
15475}
15476
15477Register fastEmit_X86ISD_FGETEXPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15478 switch (VT.SimpleTy) {
15479 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(RetVT, Op0, Op1);
15480 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(RetVT, Op0, Op1);
15481 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(RetVT, Op0, Op1);
15482 default: return Register();
15483 }
15484}
15485
15486// FastEmit functions for X86ISD::FGETEXPS_SAE.
15487
15488Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15489 if (RetVT.SimpleTy != MVT::v8f16)
15490 return Register();
15491 if ((Subtarget->hasFP16())) {
15492 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZrb, RC: &X86::VR128XRegClass, Op0, Op1);
15493 }
15494 return Register();
15495}
15496
15497Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15498 if (RetVT.SimpleTy != MVT::v4f32)
15499 return Register();
15500 if ((Subtarget->hasAVX512())) {
15501 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZrb, RC: &X86::VR128XRegClass, Op0, Op1);
15502 }
15503 return Register();
15504}
15505
15506Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15507 if (RetVT.SimpleTy != MVT::v2f64)
15508 return Register();
15509 if ((Subtarget->hasAVX512())) {
15510 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZrb, RC: &X86::VR128XRegClass, Op0, Op1);
15511 }
15512 return Register();
15513}
15514
15515Register fastEmit_X86ISD_FGETEXPS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15516 switch (VT.SimpleTy) {
15517 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
15518 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
15519 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
15520 default: return Register();
15521 }
15522}
15523
15524// FastEmit functions for X86ISD::FHADD.
15525
15526Register fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15527 if (RetVT.SimpleTy != MVT::v4f32)
15528 return Register();
15529 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15530 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15531 }
15532 if ((Subtarget->hasAVX())) {
15533 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15534 }
15535 return Register();
15536}
15537
15538Register fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15539 if (RetVT.SimpleTy != MVT::v8f32)
15540 return Register();
15541 if ((Subtarget->hasAVX())) {
15542 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15543 }
15544 return Register();
15545}
15546
15547Register fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15548 if (RetVT.SimpleTy != MVT::v2f64)
15549 return Register();
15550 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15551 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15552 }
15553 if ((Subtarget->hasAVX())) {
15554 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15555 }
15556 return Register();
15557}
15558
15559Register fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15560 if (RetVT.SimpleTy != MVT::v4f64)
15561 return Register();
15562 if ((Subtarget->hasAVX())) {
15563 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15564 }
15565 return Register();
15566}
15567
15568Register fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15569 switch (VT.SimpleTy) {
15570 case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op1);
15571 case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op1);
15572 case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op1);
15573 case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op1);
15574 default: return Register();
15575 }
15576}
15577
15578// FastEmit functions for X86ISD::FHSUB.
15579
15580Register fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15581 if (RetVT.SimpleTy != MVT::v4f32)
15582 return Register();
15583 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15584 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15585 }
15586 if ((Subtarget->hasAVX())) {
15587 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15588 }
15589 return Register();
15590}
15591
15592Register fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15593 if (RetVT.SimpleTy != MVT::v8f32)
15594 return Register();
15595 if ((Subtarget->hasAVX())) {
15596 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15597 }
15598 return Register();
15599}
15600
15601Register fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15602 if (RetVT.SimpleTy != MVT::v2f64)
15603 return Register();
15604 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15605 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15606 }
15607 if ((Subtarget->hasAVX())) {
15608 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15609 }
15610 return Register();
15611}
15612
15613Register fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15614 if (RetVT.SimpleTy != MVT::v4f64)
15615 return Register();
15616 if ((Subtarget->hasAVX())) {
15617 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15618 }
15619 return Register();
15620}
15621
15622Register fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15623 switch (VT.SimpleTy) {
15624 case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
15625 case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
15626 case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
15627 case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
15628 default: return Register();
15629 }
15630}
15631
15632// FastEmit functions for X86ISD::FMAX.
15633
15634Register fastEmit_X86ISD_FMAX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
15635 if (RetVT.SimpleTy != MVT::f16)
15636 return Register();
15637 if ((Subtarget->hasFP16())) {
15638 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
15639 }
15640 return Register();
15641}
15642
15643Register fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
15644 if (RetVT.SimpleTy != MVT::f32)
15645 return Register();
15646 if ((Subtarget->hasAVX512())) {
15647 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
15648 }
15649 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15650 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15651 }
15652 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15653 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15654 }
15655 return Register();
15656}
15657
15658Register fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
15659 if (RetVT.SimpleTy != MVT::f64)
15660 return Register();
15661 if ((Subtarget->hasAVX512())) {
15662 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
15663 }
15664 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15665 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15666 }
15667 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15668 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15669 }
15670 return Register();
15671}
15672
15673Register fastEmit_X86ISD_FMAX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15674 if (RetVT.SimpleTy != MVT::v8f16)
15675 return Register();
15676 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15677 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15678 }
15679 return Register();
15680}
15681
15682Register fastEmit_X86ISD_FMAX_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
15683 if (RetVT.SimpleTy != MVT::v16f16)
15684 return Register();
15685 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15686 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15687 }
15688 return Register();
15689}
15690
15691Register fastEmit_X86ISD_FMAX_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
15692 if (RetVT.SimpleTy != MVT::v32f16)
15693 return Register();
15694 if ((Subtarget->hasFP16())) {
15695 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
15696 }
15697 return Register();
15698}
15699
15700Register fastEmit_X86ISD_FMAX_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
15701 if (RetVT.SimpleTy != MVT::v8bf16)
15702 return Register();
15703 if ((Subtarget->hasAVX10_2())) {
15704 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15705 }
15706 return Register();
15707}
15708
15709Register fastEmit_X86ISD_FMAX_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
15710 if (RetVT.SimpleTy != MVT::v16bf16)
15711 return Register();
15712 if ((Subtarget->hasAVX10_2())) {
15713 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15714 }
15715 return Register();
15716}
15717
15718Register fastEmit_X86ISD_FMAX_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
15719 if (RetVT.SimpleTy != MVT::v32bf16)
15720 return Register();
15721 if ((Subtarget->hasAVX10_2())) {
15722 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
15723 }
15724 return Register();
15725}
15726
15727Register fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15728 if (RetVT.SimpleTy != MVT::v4f32)
15729 return Register();
15730 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15731 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15732 }
15733 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15734 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15735 }
15736 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15737 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15738 }
15739 return Register();
15740}
15741
15742Register fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15743 if (RetVT.SimpleTy != MVT::v8f32)
15744 return Register();
15745 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15746 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15747 }
15748 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15749 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15750 }
15751 return Register();
15752}
15753
15754Register fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
15755 if (RetVT.SimpleTy != MVT::v16f32)
15756 return Register();
15757 if ((Subtarget->hasAVX512())) {
15758 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
15759 }
15760 return Register();
15761}
15762
15763Register fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15764 if (RetVT.SimpleTy != MVT::v2f64)
15765 return Register();
15766 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15767 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15768 }
15769 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15770 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15771 }
15772 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15773 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15774 }
15775 return Register();
15776}
15777
15778Register fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15779 if (RetVT.SimpleTy != MVT::v4f64)
15780 return Register();
15781 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15782 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15783 }
15784 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15785 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15786 }
15787 return Register();
15788}
15789
15790Register fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
15791 if (RetVT.SimpleTy != MVT::v8f64)
15792 return Register();
15793 if ((Subtarget->hasAVX512())) {
15794 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15795 }
15796 return Register();
15797}
15798
15799Register fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15800 switch (VT.SimpleTy) {
15801 case MVT::f16: return fastEmit_X86ISD_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
15802 case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
15803 case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
15804 case MVT::v8f16: return fastEmit_X86ISD_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
15805 case MVT::v16f16: return fastEmit_X86ISD_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
15806 case MVT::v32f16: return fastEmit_X86ISD_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
15807 case MVT::v8bf16: return fastEmit_X86ISD_FMAX_MVT_v8bf16_rr(RetVT, Op0, Op1);
15808 case MVT::v16bf16: return fastEmit_X86ISD_FMAX_MVT_v16bf16_rr(RetVT, Op0, Op1);
15809 case MVT::v32bf16: return fastEmit_X86ISD_FMAX_MVT_v32bf16_rr(RetVT, Op0, Op1);
15810 case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
15811 case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
15812 case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
15813 case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
15814 case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
15815 case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
15816 default: return Register();
15817 }
15818}
15819
15820// FastEmit functions for X86ISD::FMAXC.
15821
15822Register fastEmit_X86ISD_FMAXC_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
15823 if (RetVT.SimpleTy != MVT::f16)
15824 return Register();
15825 if ((Subtarget->hasAVX512())) {
15826 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
15827 }
15828 return Register();
15829}
15830
15831Register fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
15832 if (RetVT.SimpleTy != MVT::f32)
15833 return Register();
15834 if ((Subtarget->hasAVX512())) {
15835 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
15836 }
15837 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15838 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15839 }
15840 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15841 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15842 }
15843 return Register();
15844}
15845
15846Register fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
15847 if (RetVT.SimpleTy != MVT::f64)
15848 return Register();
15849 if ((Subtarget->hasAVX512())) {
15850 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
15851 }
15852 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15853 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15854 }
15855 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15856 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15857 }
15858 return Register();
15859}
15860
15861Register fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15862 if (RetVT.SimpleTy != MVT::v8f16)
15863 return Register();
15864 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15865 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15866 }
15867 return Register();
15868}
15869
15870Register fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
15871 if (RetVT.SimpleTy != MVT::v16f16)
15872 return Register();
15873 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15874 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15875 }
15876 return Register();
15877}
15878
15879Register fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
15880 if (RetVT.SimpleTy != MVT::v32f16)
15881 return Register();
15882 if ((Subtarget->hasFP16())) {
15883 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
15884 }
15885 return Register();
15886}
15887
15888Register fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15889 if (RetVT.SimpleTy != MVT::v4f32)
15890 return Register();
15891 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15892 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15893 }
15894 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15895 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15896 }
15897 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15898 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15899 }
15900 return Register();
15901}
15902
15903Register fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15904 if (RetVT.SimpleTy != MVT::v8f32)
15905 return Register();
15906 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15907 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15908 }
15909 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15910 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15911 }
15912 return Register();
15913}
15914
15915Register fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
15916 if (RetVT.SimpleTy != MVT::v16f32)
15917 return Register();
15918 if ((Subtarget->hasAVX512())) {
15919 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
15920 }
15921 return Register();
15922}
15923
15924Register fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15925 if (RetVT.SimpleTy != MVT::v2f64)
15926 return Register();
15927 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15928 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15929 }
15930 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15931 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15932 }
15933 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15934 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15935 }
15936 return Register();
15937}
15938
15939Register fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15940 if (RetVT.SimpleTy != MVT::v4f64)
15941 return Register();
15942 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15943 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15944 }
15945 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15946 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15947 }
15948 return Register();
15949}
15950
15951Register fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
15952 if (RetVT.SimpleTy != MVT::v8f64)
15953 return Register();
15954 if ((Subtarget->hasAVX512())) {
15955 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15956 }
15957 return Register();
15958}
15959
15960Register fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15961 switch (VT.SimpleTy) {
15962 case MVT::f16: return fastEmit_X86ISD_FMAXC_MVT_f16_rr(RetVT, Op0, Op1);
15963 case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op1);
15964 case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op1);
15965 case MVT::v8f16: return fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(RetVT, Op0, Op1);
15966 case MVT::v16f16: return fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(RetVT, Op0, Op1);
15967 case MVT::v32f16: return fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(RetVT, Op0, Op1);
15968 case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op1);
15969 case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op1);
15970 case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op1);
15971 case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op1);
15972 case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op1);
15973 case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op1);
15974 default: return Register();
15975 }
15976}
15977
15978// FastEmit functions for X86ISD::FMAXS.
15979
15980Register fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15981 if (RetVT.SimpleTy != MVT::v8f16)
15982 return Register();
15983 if ((Subtarget->hasFP16())) {
15984 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15985 }
15986 return Register();
15987}
15988
15989Register fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15990 if (RetVT.SimpleTy != MVT::v4f32)
15991 return Register();
15992 if ((Subtarget->hasAVX512())) {
15993 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15994 }
15995 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15996 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15997 }
15998 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15999 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16000 }
16001 return Register();
16002}
16003
16004Register fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16005 if (RetVT.SimpleTy != MVT::v2f64)
16006 return Register();
16007 if ((Subtarget->hasAVX512())) {
16008 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16009 }
16010 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16011 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16012 }
16013 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16014 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16015 }
16016 return Register();
16017}
16018
16019Register fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16020 switch (VT.SimpleTy) {
16021 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(RetVT, Op0, Op1);
16022 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op1);
16023 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op1);
16024 default: return Register();
16025 }
16026}
16027
16028// FastEmit functions for X86ISD::FMAXS_SAE.
16029
16030Register fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16031 if (RetVT.SimpleTy != MVT::v8f16)
16032 return Register();
16033 if ((Subtarget->hasFP16())) {
16034 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16035 }
16036 return Register();
16037}
16038
16039Register fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16040 if (RetVT.SimpleTy != MVT::v4f32)
16041 return Register();
16042 if ((Subtarget->hasAVX512())) {
16043 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16044 }
16045 return Register();
16046}
16047
16048Register fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16049 if (RetVT.SimpleTy != MVT::v2f64)
16050 return Register();
16051 if ((Subtarget->hasAVX512())) {
16052 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16053 }
16054 return Register();
16055}
16056
16057Register fastEmit_X86ISD_FMAXS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16058 switch (VT.SimpleTy) {
16059 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
16060 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16061 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16062 default: return Register();
16063 }
16064}
16065
16066// FastEmit functions for X86ISD::FMAX_SAE.
16067
16068Register fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16069 if (RetVT.SimpleTy != MVT::v32f16)
16070 return Register();
16071 if ((Subtarget->hasFP16())) {
16072 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16073 }
16074 return Register();
16075}
16076
16077Register fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16078 if (RetVT.SimpleTy != MVT::v16f32)
16079 return Register();
16080 if ((Subtarget->hasAVX512())) {
16081 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16082 }
16083 return Register();
16084}
16085
16086Register fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16087 if (RetVT.SimpleTy != MVT::v8f64)
16088 return Register();
16089 if ((Subtarget->hasAVX512())) {
16090 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16091 }
16092 return Register();
16093}
16094
16095Register fastEmit_X86ISD_FMAX_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16096 switch (VT.SimpleTy) {
16097 case MVT::v32f16: return fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
16098 case MVT::v16f32: return fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
16099 case MVT::v8f64: return fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
16100 default: return Register();
16101 }
16102}
16103
16104// FastEmit functions for X86ISD::FMIN.
16105
16106Register fastEmit_X86ISD_FMIN_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16107 if (RetVT.SimpleTy != MVT::f16)
16108 return Register();
16109 if ((Subtarget->hasFP16())) {
16110 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16111 }
16112 return Register();
16113}
16114
16115Register fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16116 if (RetVT.SimpleTy != MVT::f32)
16117 return Register();
16118 if ((Subtarget->hasAVX512())) {
16119 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16120 }
16121 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16122 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16123 }
16124 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16125 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16126 }
16127 return Register();
16128}
16129
16130Register fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16131 if (RetVT.SimpleTy != MVT::f64)
16132 return Register();
16133 if ((Subtarget->hasAVX512())) {
16134 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16135 }
16136 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16137 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16138 }
16139 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16140 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16141 }
16142 return Register();
16143}
16144
16145Register fastEmit_X86ISD_FMIN_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16146 if (RetVT.SimpleTy != MVT::v8f16)
16147 return Register();
16148 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16149 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16150 }
16151 return Register();
16152}
16153
16154Register fastEmit_X86ISD_FMIN_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
16155 if (RetVT.SimpleTy != MVT::v16f16)
16156 return Register();
16157 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16158 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16159 }
16160 return Register();
16161}
16162
16163Register fastEmit_X86ISD_FMIN_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16164 if (RetVT.SimpleTy != MVT::v32f16)
16165 return Register();
16166 if ((Subtarget->hasFP16())) {
16167 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16168 }
16169 return Register();
16170}
16171
16172Register fastEmit_X86ISD_FMIN_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
16173 if (RetVT.SimpleTy != MVT::v8bf16)
16174 return Register();
16175 if ((Subtarget->hasAVX10_2())) {
16176 return fastEmitInst_rr(MachineInstOpcode: X86::VMINBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16177 }
16178 return Register();
16179}
16180
16181Register fastEmit_X86ISD_FMIN_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
16182 if (RetVT.SimpleTy != MVT::v16bf16)
16183 return Register();
16184 if ((Subtarget->hasAVX10_2())) {
16185 return fastEmitInst_rr(MachineInstOpcode: X86::VMINBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16186 }
16187 return Register();
16188}
16189
16190Register fastEmit_X86ISD_FMIN_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
16191 if (RetVT.SimpleTy != MVT::v32bf16)
16192 return Register();
16193 if ((Subtarget->hasAVX10_2())) {
16194 return fastEmitInst_rr(MachineInstOpcode: X86::VMINBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
16195 }
16196 return Register();
16197}
16198
16199Register fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16200 if (RetVT.SimpleTy != MVT::v4f32)
16201 return Register();
16202 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16203 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16204 }
16205 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16206 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16207 }
16208 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16209 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16210 }
16211 return Register();
16212}
16213
16214Register fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16215 if (RetVT.SimpleTy != MVT::v8f32)
16216 return Register();
16217 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16218 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16219 }
16220 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16221 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16222 }
16223 return Register();
16224}
16225
16226Register fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16227 if (RetVT.SimpleTy != MVT::v16f32)
16228 return Register();
16229 if ((Subtarget->hasAVX512())) {
16230 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16231 }
16232 return Register();
16233}
16234
16235Register fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16236 if (RetVT.SimpleTy != MVT::v2f64)
16237 return Register();
16238 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16239 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16240 }
16241 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16242 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16243 }
16244 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16245 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16246 }
16247 return Register();
16248}
16249
16250Register fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16251 if (RetVT.SimpleTy != MVT::v4f64)
16252 return Register();
16253 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16254 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16255 }
16256 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16257 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16258 }
16259 return Register();
16260}
16261
16262Register fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16263 if (RetVT.SimpleTy != MVT::v8f64)
16264 return Register();
16265 if ((Subtarget->hasAVX512())) {
16266 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16267 }
16268 return Register();
16269}
16270
16271Register fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16272 switch (VT.SimpleTy) {
16273 case MVT::f16: return fastEmit_X86ISD_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
16274 case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
16275 case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
16276 case MVT::v8f16: return fastEmit_X86ISD_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
16277 case MVT::v16f16: return fastEmit_X86ISD_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
16278 case MVT::v32f16: return fastEmit_X86ISD_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
16279 case MVT::v8bf16: return fastEmit_X86ISD_FMIN_MVT_v8bf16_rr(RetVT, Op0, Op1);
16280 case MVT::v16bf16: return fastEmit_X86ISD_FMIN_MVT_v16bf16_rr(RetVT, Op0, Op1);
16281 case MVT::v32bf16: return fastEmit_X86ISD_FMIN_MVT_v32bf16_rr(RetVT, Op0, Op1);
16282 case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
16283 case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
16284 case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
16285 case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
16286 case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
16287 case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
16288 default: return Register();
16289 }
16290}
16291
16292// FastEmit functions for X86ISD::FMINC.
16293
16294Register fastEmit_X86ISD_FMINC_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16295 if (RetVT.SimpleTy != MVT::f16)
16296 return Register();
16297 if ((Subtarget->hasAVX512())) {
16298 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16299 }
16300 return Register();
16301}
16302
16303Register fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16304 if (RetVT.SimpleTy != MVT::f32)
16305 return Register();
16306 if ((Subtarget->hasAVX512())) {
16307 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16308 }
16309 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16310 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16311 }
16312 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16313 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16314 }
16315 return Register();
16316}
16317
16318Register fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16319 if (RetVT.SimpleTy != MVT::f64)
16320 return Register();
16321 if ((Subtarget->hasAVX512())) {
16322 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16323 }
16324 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16325 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16326 }
16327 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16328 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16329 }
16330 return Register();
16331}
16332
16333Register fastEmit_X86ISD_FMINC_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16334 if (RetVT.SimpleTy != MVT::v8f16)
16335 return Register();
16336 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16337 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16338 }
16339 return Register();
16340}
16341
16342Register fastEmit_X86ISD_FMINC_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
16343 if (RetVT.SimpleTy != MVT::v16f16)
16344 return Register();
16345 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16346 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16347 }
16348 return Register();
16349}
16350
16351Register fastEmit_X86ISD_FMINC_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16352 if (RetVT.SimpleTy != MVT::v32f16)
16353 return Register();
16354 if ((Subtarget->hasFP16())) {
16355 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16356 }
16357 return Register();
16358}
16359
16360Register fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16361 if (RetVT.SimpleTy != MVT::v4f32)
16362 return Register();
16363 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16364 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16365 }
16366 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16367 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16368 }
16369 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16370 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16371 }
16372 return Register();
16373}
16374
16375Register fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16376 if (RetVT.SimpleTy != MVT::v8f32)
16377 return Register();
16378 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16379 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16380 }
16381 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16382 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16383 }
16384 return Register();
16385}
16386
16387Register fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16388 if (RetVT.SimpleTy != MVT::v16f32)
16389 return Register();
16390 if ((Subtarget->hasAVX512())) {
16391 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16392 }
16393 return Register();
16394}
16395
16396Register fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16397 if (RetVT.SimpleTy != MVT::v2f64)
16398 return Register();
16399 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16400 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16401 }
16402 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16403 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16404 }
16405 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16406 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16407 }
16408 return Register();
16409}
16410
16411Register fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16412 if (RetVT.SimpleTy != MVT::v4f64)
16413 return Register();
16414 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16415 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16416 }
16417 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16418 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16419 }
16420 return Register();
16421}
16422
16423Register fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16424 if (RetVT.SimpleTy != MVT::v8f64)
16425 return Register();
16426 if ((Subtarget->hasAVX512())) {
16427 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16428 }
16429 return Register();
16430}
16431
16432Register fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16433 switch (VT.SimpleTy) {
16434 case MVT::f16: return fastEmit_X86ISD_FMINC_MVT_f16_rr(RetVT, Op0, Op1);
16435 case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op1);
16436 case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op1);
16437 case MVT::v8f16: return fastEmit_X86ISD_FMINC_MVT_v8f16_rr(RetVT, Op0, Op1);
16438 case MVT::v16f16: return fastEmit_X86ISD_FMINC_MVT_v16f16_rr(RetVT, Op0, Op1);
16439 case MVT::v32f16: return fastEmit_X86ISD_FMINC_MVT_v32f16_rr(RetVT, Op0, Op1);
16440 case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op1);
16441 case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op1);
16442 case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op1);
16443 case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op1);
16444 case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op1);
16445 case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op1);
16446 default: return Register();
16447 }
16448}
16449
16450// FastEmit functions for X86ISD::FMINS.
16451
16452Register fastEmit_X86ISD_FMINS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16453 if (RetVT.SimpleTy != MVT::v8f16)
16454 return Register();
16455 if ((Subtarget->hasFP16())) {
16456 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16457 }
16458 return Register();
16459}
16460
16461Register fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16462 if (RetVT.SimpleTy != MVT::v4f32)
16463 return Register();
16464 if ((Subtarget->hasAVX512())) {
16465 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16466 }
16467 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16468 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16469 }
16470 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16471 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16472 }
16473 return Register();
16474}
16475
16476Register fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16477 if (RetVT.SimpleTy != MVT::v2f64)
16478 return Register();
16479 if ((Subtarget->hasAVX512())) {
16480 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16481 }
16482 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16483 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16484 }
16485 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16486 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16487 }
16488 return Register();
16489}
16490
16491Register fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16492 switch (VT.SimpleTy) {
16493 case MVT::v8f16: return fastEmit_X86ISD_FMINS_MVT_v8f16_rr(RetVT, Op0, Op1);
16494 case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op1);
16495 case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op1);
16496 default: return Register();
16497 }
16498}
16499
16500// FastEmit functions for X86ISD::FMINS_SAE.
16501
16502Register fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16503 if (RetVT.SimpleTy != MVT::v8f16)
16504 return Register();
16505 if ((Subtarget->hasFP16())) {
16506 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16507 }
16508 return Register();
16509}
16510
16511Register fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16512 if (RetVT.SimpleTy != MVT::v4f32)
16513 return Register();
16514 if ((Subtarget->hasAVX512())) {
16515 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16516 }
16517 return Register();
16518}
16519
16520Register fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16521 if (RetVT.SimpleTy != MVT::v2f64)
16522 return Register();
16523 if ((Subtarget->hasAVX512())) {
16524 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16525 }
16526 return Register();
16527}
16528
16529Register fastEmit_X86ISD_FMINS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16530 switch (VT.SimpleTy) {
16531 case MVT::v8f16: return fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
16532 case MVT::v4f32: return fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16533 case MVT::v2f64: return fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16534 default: return Register();
16535 }
16536}
16537
16538// FastEmit functions for X86ISD::FMIN_SAE.
16539
16540Register fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16541 if (RetVT.SimpleTy != MVT::v32f16)
16542 return Register();
16543 if ((Subtarget->hasFP16())) {
16544 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16545 }
16546 return Register();
16547}
16548
16549Register fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16550 if (RetVT.SimpleTy != MVT::v16f32)
16551 return Register();
16552 if ((Subtarget->hasAVX512())) {
16553 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16554 }
16555 return Register();
16556}
16557
16558Register fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16559 if (RetVT.SimpleTy != MVT::v8f64)
16560 return Register();
16561 if ((Subtarget->hasAVX512())) {
16562 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16563 }
16564 return Register();
16565}
16566
16567Register fastEmit_X86ISD_FMIN_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16568 switch (VT.SimpleTy) {
16569 case MVT::v32f16: return fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
16570 case MVT::v16f32: return fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
16571 case MVT::v8f64: return fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
16572 default: return Register();
16573 }
16574}
16575
16576// FastEmit functions for X86ISD::FMULS.
16577
16578Register fastEmit_X86ISD_FMULS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16579 if (RetVT.SimpleTy != MVT::v8f16)
16580 return Register();
16581 if ((Subtarget->hasFP16())) {
16582 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16583 }
16584 return Register();
16585}
16586
16587Register fastEmit_X86ISD_FMULS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16588 if (RetVT.SimpleTy != MVT::v4f32)
16589 return Register();
16590 if ((Subtarget->hasAVX512())) {
16591 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16592 }
16593 return Register();
16594}
16595
16596Register fastEmit_X86ISD_FMULS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16597 if (RetVT.SimpleTy != MVT::v2f64)
16598 return Register();
16599 if ((Subtarget->hasAVX512())) {
16600 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16601 }
16602 return Register();
16603}
16604
16605Register fastEmit_X86ISD_FMULS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16606 switch (VT.SimpleTy) {
16607 case MVT::v8f16: return fastEmit_X86ISD_FMULS_MVT_v8f16_rr(RetVT, Op0, Op1);
16608 case MVT::v4f32: return fastEmit_X86ISD_FMULS_MVT_v4f32_rr(RetVT, Op0, Op1);
16609 case MVT::v2f64: return fastEmit_X86ISD_FMULS_MVT_v2f64_rr(RetVT, Op0, Op1);
16610 default: return Register();
16611 }
16612}
16613
16614// FastEmit functions for X86ISD::FOR.
16615
16616Register fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
16617 if (RetVT.SimpleTy != MVT::f128)
16618 return Register();
16619 if ((Subtarget->hasVLX())) {
16620 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16621 }
16622 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16623 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16624 }
16625 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16626 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16627 }
16628 return Register();
16629}
16630
16631Register fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16632 if (RetVT.SimpleTy != MVT::v4f32)
16633 return Register();
16634 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16635}
16636
16637Register fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16638 switch (VT.SimpleTy) {
16639 case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op1);
16640 case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op1);
16641 default: return Register();
16642 }
16643}
16644
16645// FastEmit functions for X86ISD::FP80_ADD.
16646
16647Register fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
16648 if (RetVT.SimpleTy != MVT::f80)
16649 return Register();
16650 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
16651}
16652
16653Register fastEmit_X86ISD_FP80_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16654 switch (VT.SimpleTy) {
16655 case MVT::f80: return fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
16656 default: return Register();
16657 }
16658}
16659
16660// FastEmit functions for X86ISD::FSQRTS.
16661
16662Register fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16663 if (RetVT.SimpleTy != MVT::v8f16)
16664 return Register();
16665 if ((Subtarget->hasFP16())) {
16666 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSHZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16667 }
16668 return Register();
16669}
16670
16671Register fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16672 if (RetVT.SimpleTy != MVT::v4f32)
16673 return Register();
16674 if ((Subtarget->hasAVX512())) {
16675 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSSZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16676 }
16677 return Register();
16678}
16679
16680Register fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16681 if (RetVT.SimpleTy != MVT::v2f64)
16682 return Register();
16683 if ((Subtarget->hasAVX512())) {
16684 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSDZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16685 }
16686 return Register();
16687}
16688
16689Register fastEmit_X86ISD_FSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16690 switch (VT.SimpleTy) {
16691 case MVT::v8f16: return fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(RetVT, Op0, Op1);
16692 case MVT::v4f32: return fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
16693 case MVT::v2f64: return fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
16694 default: return Register();
16695 }
16696}
16697
16698// FastEmit functions for X86ISD::FSUBS.
16699
16700Register fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16701 if (RetVT.SimpleTy != MVT::v8f16)
16702 return Register();
16703 if ((Subtarget->hasFP16())) {
16704 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16705 }
16706 return Register();
16707}
16708
16709Register fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16710 if (RetVT.SimpleTy != MVT::v4f32)
16711 return Register();
16712 if ((Subtarget->hasAVX512())) {
16713 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16714 }
16715 return Register();
16716}
16717
16718Register fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16719 if (RetVT.SimpleTy != MVT::v2f64)
16720 return Register();
16721 if ((Subtarget->hasAVX512())) {
16722 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16723 }
16724 return Register();
16725}
16726
16727Register fastEmit_X86ISD_FSUBS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16728 switch (VT.SimpleTy) {
16729 case MVT::v8f16: return fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(RetVT, Op0, Op1);
16730 case MVT::v4f32: return fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(RetVT, Op0, Op1);
16731 case MVT::v2f64: return fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(RetVT, Op0, Op1);
16732 default: return Register();
16733 }
16734}
16735
16736// FastEmit functions for X86ISD::FXOR.
16737
16738Register fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
16739 if (RetVT.SimpleTy != MVT::f128)
16740 return Register();
16741 if ((Subtarget->hasVLX())) {
16742 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16743 }
16744 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16745 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16746 }
16747 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16748 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16749 }
16750 return Register();
16751}
16752
16753Register fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16754 if (RetVT.SimpleTy != MVT::v4f32)
16755 return Register();
16756 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16757}
16758
16759Register fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16760 switch (VT.SimpleTy) {
16761 case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op1);
16762 case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op1);
16763 default: return Register();
16764 }
16765}
16766
16767// FastEmit functions for X86ISD::GF2P8MULB.
16768
16769Register fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
16770 if (RetVT.SimpleTy != MVT::v16i8)
16771 return Register();
16772 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
16773 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16774 }
16775 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
16776 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
16777 }
16778 if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16779 return fastEmitInst_rr(MachineInstOpcode: X86::GF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
16780 }
16781 return Register();
16782}
16783
16784Register fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
16785 if (RetVT.SimpleTy != MVT::v32i8)
16786 return Register();
16787 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
16788 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16789 }
16790 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
16791 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBYrr, RC: &X86::VR256RegClass, Op0, Op1);
16792 }
16793 return Register();
16794}
16795
16796Register fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
16797 if (RetVT.SimpleTy != MVT::v64i8)
16798 return Register();
16799 if ((Subtarget->hasAVX512()) && (Subtarget->hasGFNI())) {
16800 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZrr, RC: &X86::VR512RegClass, Op0, Op1);
16801 }
16802 return Register();
16803}
16804
16805Register fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16806 switch (VT.SimpleTy) {
16807 case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op1);
16808 case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op1);
16809 case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op1);
16810 default: return Register();
16811 }
16812}
16813
16814// FastEmit functions for X86ISD::HADD.
16815
16816Register fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
16817 if (RetVT.SimpleTy != MVT::v8i16)
16818 return Register();
16819 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16820 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
16821 }
16822 if ((Subtarget->hasAVX())) {
16823 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
16824 }
16825 return Register();
16826}
16827
16828Register fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
16829 if (RetVT.SimpleTy != MVT::v16i16)
16830 return Register();
16831 if ((Subtarget->hasAVX2())) {
16832 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16833 }
16834 return Register();
16835}
16836
16837Register fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
16838 if (RetVT.SimpleTy != MVT::v4i32)
16839 return Register();
16840 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16841 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
16842 }
16843 if ((Subtarget->hasAVX())) {
16844 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
16845 }
16846 return Register();
16847}
16848
16849Register fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
16850 if (RetVT.SimpleTy != MVT::v8i32)
16851 return Register();
16852 if ((Subtarget->hasAVX2())) {
16853 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16854 }
16855 return Register();
16856}
16857
16858Register fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16859 switch (VT.SimpleTy) {
16860 case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op1);
16861 case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op1);
16862 case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op1);
16863 case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op1);
16864 default: return Register();
16865 }
16866}
16867
16868// FastEmit functions for X86ISD::HADDS.
16869
16870Register fastEmit_X86ISD_HADDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
16871 if (RetVT.SimpleTy != MVT::v8i16)
16872 return Register();
16873 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16874 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
16875 }
16876 if ((Subtarget->hasAVX())) {
16877 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
16878 }
16879 return Register();
16880}
16881
16882Register fastEmit_X86ISD_HADDS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
16883 if (RetVT.SimpleTy != MVT::v16i16)
16884 return Register();
16885 if ((Subtarget->hasAVX2())) {
16886 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16887 }
16888 return Register();
16889}
16890
16891Register fastEmit_X86ISD_HADDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16892 switch (VT.SimpleTy) {
16893 case MVT::v8i16: return fastEmit_X86ISD_HADDS_MVT_v8i16_rr(RetVT, Op0, Op1);
16894 case MVT::v16i16: return fastEmit_X86ISD_HADDS_MVT_v16i16_rr(RetVT, Op0, Op1);
16895 default: return Register();
16896 }
16897}
16898
16899// FastEmit functions for X86ISD::HSUB.
16900
16901Register fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
16902 if (RetVT.SimpleTy != MVT::v8i16)
16903 return Register();
16904 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16905 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
16906 }
16907 if ((Subtarget->hasAVX())) {
16908 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
16909 }
16910 return Register();
16911}
16912
16913Register fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
16914 if (RetVT.SimpleTy != MVT::v16i16)
16915 return Register();
16916 if ((Subtarget->hasAVX2())) {
16917 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16918 }
16919 return Register();
16920}
16921
16922Register fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
16923 if (RetVT.SimpleTy != MVT::v4i32)
16924 return Register();
16925 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16926 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
16927 }
16928 if ((Subtarget->hasAVX())) {
16929 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
16930 }
16931 return Register();
16932}
16933
16934Register fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
16935 if (RetVT.SimpleTy != MVT::v8i32)
16936 return Register();
16937 if ((Subtarget->hasAVX2())) {
16938 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16939 }
16940 return Register();
16941}
16942
16943Register fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16944 switch (VT.SimpleTy) {
16945 case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op1);
16946 case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op1);
16947 case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op1);
16948 case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op1);
16949 default: return Register();
16950 }
16951}
16952
16953// FastEmit functions for X86ISD::HSUBS.
16954
16955Register fastEmit_X86ISD_HSUBS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
16956 if (RetVT.SimpleTy != MVT::v8i16)
16957 return Register();
16958 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16959 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
16960 }
16961 if ((Subtarget->hasAVX())) {
16962 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
16963 }
16964 return Register();
16965}
16966
16967Register fastEmit_X86ISD_HSUBS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
16968 if (RetVT.SimpleTy != MVT::v16i16)
16969 return Register();
16970 if ((Subtarget->hasAVX2())) {
16971 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16972 }
16973 return Register();
16974}
16975
16976Register fastEmit_X86ISD_HSUBS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16977 switch (VT.SimpleTy) {
16978 case MVT::v8i16: return fastEmit_X86ISD_HSUBS_MVT_v8i16_rr(RetVT, Op0, Op1);
16979 case MVT::v16i16: return fastEmit_X86ISD_HSUBS_MVT_v16i16_rr(RetVT, Op0, Op1);
16980 default: return Register();
16981 }
16982}
16983
16984// FastEmit functions for X86ISD::KADD.
16985
16986Register fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
16987 if (RetVT.SimpleTy != MVT::v8i1)
16988 return Register();
16989 if ((Subtarget->hasDQI())) {
16990 return fastEmitInst_rr(MachineInstOpcode: X86::KADDBkk, RC: &X86::VK8RegClass, Op0, Op1);
16991 }
16992 return Register();
16993}
16994
16995Register fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
16996 if (RetVT.SimpleTy != MVT::v16i1)
16997 return Register();
16998 if ((Subtarget->hasDQI())) {
16999 return fastEmitInst_rr(MachineInstOpcode: X86::KADDWkk, RC: &X86::VK16RegClass, Op0, Op1);
17000 }
17001 return Register();
17002}
17003
17004Register fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
17005 if (RetVT.SimpleTy != MVT::v32i1)
17006 return Register();
17007 if ((Subtarget->hasBWI())) {
17008 return fastEmitInst_rr(MachineInstOpcode: X86::KADDDkk, RC: &X86::VK32RegClass, Op0, Op1);
17009 }
17010 return Register();
17011}
17012
17013Register fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
17014 if (RetVT.SimpleTy != MVT::v64i1)
17015 return Register();
17016 if ((Subtarget->hasBWI())) {
17017 return fastEmitInst_rr(MachineInstOpcode: X86::KADDQkk, RC: &X86::VK64RegClass, Op0, Op1);
17018 }
17019 return Register();
17020}
17021
17022Register fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17023 switch (VT.SimpleTy) {
17024 case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op1);
17025 case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op1);
17026 case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op1);
17027 case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op1);
17028 default: return Register();
17029 }
17030}
17031
17032// FastEmit functions for X86ISD::KORTEST.
17033
17034Register fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
17035 if (RetVT.SimpleTy != MVT::i32)
17036 return Register();
17037 if ((Subtarget->hasDQI())) {
17038 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTBkk, RC: &X86::VK8RegClass, Op0, Op1);
17039 }
17040 return Register();
17041}
17042
17043Register fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
17044 if (RetVT.SimpleTy != MVT::i32)
17045 return Register();
17046 if ((Subtarget->hasAVX512())) {
17047 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTWkk, RC: &X86::VK16RegClass, Op0, Op1);
17048 }
17049 return Register();
17050}
17051
17052Register fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
17053 if (RetVT.SimpleTy != MVT::i32)
17054 return Register();
17055 if ((Subtarget->hasBWI())) {
17056 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTDkk, RC: &X86::VK32RegClass, Op0, Op1);
17057 }
17058 return Register();
17059}
17060
17061Register fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
17062 if (RetVT.SimpleTy != MVT::i32)
17063 return Register();
17064 if ((Subtarget->hasBWI())) {
17065 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTQkk, RC: &X86::VK64RegClass, Op0, Op1);
17066 }
17067 return Register();
17068}
17069
17070Register fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17071 switch (VT.SimpleTy) {
17072 case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
17073 case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
17074 case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
17075 case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
17076 default: return Register();
17077 }
17078}
17079
17080// FastEmit functions for X86ISD::KTEST.
17081
17082Register fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
17083 if (RetVT.SimpleTy != MVT::i32)
17084 return Register();
17085 if ((Subtarget->hasDQI())) {
17086 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTBkk, RC: &X86::VK8RegClass, Op0, Op1);
17087 }
17088 return Register();
17089}
17090
17091Register fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
17092 if (RetVT.SimpleTy != MVT::i32)
17093 return Register();
17094 if ((Subtarget->hasDQI())) {
17095 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTWkk, RC: &X86::VK16RegClass, Op0, Op1);
17096 }
17097 return Register();
17098}
17099
17100Register fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
17101 if (RetVT.SimpleTy != MVT::i32)
17102 return Register();
17103 if ((Subtarget->hasBWI())) {
17104 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTDkk, RC: &X86::VK32RegClass, Op0, Op1);
17105 }
17106 return Register();
17107}
17108
17109Register fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
17110 if (RetVT.SimpleTy != MVT::i32)
17111 return Register();
17112 if ((Subtarget->hasBWI())) {
17113 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTQkk, RC: &X86::VK64RegClass, Op0, Op1);
17114 }
17115 return Register();
17116}
17117
17118Register fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17119 switch (VT.SimpleTy) {
17120 case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
17121 case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
17122 case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
17123 case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
17124 default: return Register();
17125 }
17126}
17127
17128// FastEmit functions for X86ISD::MOVHLPS.
17129
17130Register fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17131 if (RetVT.SimpleTy != MVT::v4f32)
17132 return Register();
17133 if ((Subtarget->hasAVX512())) {
17134 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17135 }
17136 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17137 return fastEmitInst_rr(MachineInstOpcode: X86::MOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17138 }
17139 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17140 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17141 }
17142 return Register();
17143}
17144
17145Register fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17146 switch (VT.SimpleTy) {
17147 case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op1);
17148 default: return Register();
17149 }
17150}
17151
17152// FastEmit functions for X86ISD::MOVLHPS.
17153
17154Register fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17155 if (RetVT.SimpleTy != MVT::v4f32)
17156 return Register();
17157 if ((Subtarget->hasAVX512())) {
17158 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17159 }
17160 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17161 return fastEmitInst_rr(MachineInstOpcode: X86::MOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17162 }
17163 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17164 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17165 }
17166 return Register();
17167}
17168
17169Register fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17170 switch (VT.SimpleTy) {
17171 case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op1);
17172 default: return Register();
17173 }
17174}
17175
17176// FastEmit functions for X86ISD::MOVSD.
17177
17178Register fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17179 if (RetVT.SimpleTy != MVT::v2f64)
17180 return Register();
17181 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
17182 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17183 }
17184 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17185 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
17186 }
17187 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17188 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
17189 }
17190 return Register();
17191}
17192
17193Register fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17194 switch (VT.SimpleTy) {
17195 case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op1);
17196 default: return Register();
17197 }
17198}
17199
17200// FastEmit functions for X86ISD::MOVSH.
17201
17202Register fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17203 if (RetVT.SimpleTy != MVT::v8f16)
17204 return Register();
17205 if ((Subtarget->hasFP16())) {
17206 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17207 }
17208 return Register();
17209}
17210
17211Register fastEmit_X86ISD_MOVSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17212 switch (VT.SimpleTy) {
17213 case MVT::v8f16: return fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(RetVT, Op0, Op1);
17214 default: return Register();
17215 }
17216}
17217
17218// FastEmit functions for X86ISD::MOVSS.
17219
17220Register fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17221 if (RetVT.SimpleTy != MVT::v4f32)
17222 return Register();
17223 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
17224 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17225 }
17226 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17227 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
17228 }
17229 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17230 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
17231 }
17232 return Register();
17233}
17234
17235Register fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17236 switch (VT.SimpleTy) {
17237 case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op1);
17238 default: return Register();
17239 }
17240}
17241
17242// FastEmit functions for X86ISD::MULHRS.
17243
17244Register fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17245 if (RetVT.SimpleTy != MVT::v8i16)
17246 return Register();
17247 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17248 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17249 }
17250 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17251 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17252 }
17253 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17254 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17255 }
17256 return Register();
17257}
17258
17259Register fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17260 if (RetVT.SimpleTy != MVT::v16i16)
17261 return Register();
17262 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17263 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17264 }
17265 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17266 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17267 }
17268 return Register();
17269}
17270
17271Register fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17272 if (RetVT.SimpleTy != MVT::v32i16)
17273 return Register();
17274 if ((Subtarget->hasBWI())) {
17275 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17276 }
17277 return Register();
17278}
17279
17280Register fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17281 switch (VT.SimpleTy) {
17282 case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op1);
17283 case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op1);
17284 case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op1);
17285 default: return Register();
17286 }
17287}
17288
17289// FastEmit functions for X86ISD::MULTISHIFT.
17290
17291Register fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17292 if (RetVT.SimpleTy != MVT::v16i8)
17293 return Register();
17294 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17295 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17296 }
17297 return Register();
17298}
17299
17300Register fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17301 if (RetVT.SimpleTy != MVT::v32i8)
17302 return Register();
17303 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17304 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17305 }
17306 return Register();
17307}
17308
17309Register fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17310 if (RetVT.SimpleTy != MVT::v64i8)
17311 return Register();
17312 if ((Subtarget->hasVBMI())) {
17313 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17314 }
17315 return Register();
17316}
17317
17318Register fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17319 switch (VT.SimpleTy) {
17320 case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op1);
17321 case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op1);
17322 case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op1);
17323 default: return Register();
17324 }
17325}
17326
17327// FastEmit functions for X86ISD::PACKSS.
17328
17329Register fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17330 if (RetVT.SimpleTy != MVT::v16i8)
17331 return Register();
17332 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17333 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17334 }
17335 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17336 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17337 }
17338 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17339 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17340 }
17341 return Register();
17342}
17343
17344Register fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17345 if (RetVT.SimpleTy != MVT::v32i8)
17346 return Register();
17347 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17348 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17349 }
17350 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17351 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17352 }
17353 return Register();
17354}
17355
17356Register fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17357 if (RetVT.SimpleTy != MVT::v64i8)
17358 return Register();
17359 if ((Subtarget->hasBWI())) {
17360 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17361 }
17362 return Register();
17363}
17364
17365Register fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17366 if (RetVT.SimpleTy != MVT::v8i16)
17367 return Register();
17368 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17369 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17370 }
17371 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17372 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17373 }
17374 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17375 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17376 }
17377 return Register();
17378}
17379
17380Register fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17381 if (RetVT.SimpleTy != MVT::v16i16)
17382 return Register();
17383 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17384 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17385 }
17386 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17387 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17388 }
17389 return Register();
17390}
17391
17392Register fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
17393 if (RetVT.SimpleTy != MVT::v32i16)
17394 return Register();
17395 if ((Subtarget->hasBWI())) {
17396 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17397 }
17398 return Register();
17399}
17400
17401Register fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17402 switch (VT.SimpleTy) {
17403 case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op1);
17404 case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op1);
17405 case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op1);
17406 case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op1);
17407 case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op1);
17408 case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op1);
17409 default: return Register();
17410 }
17411}
17412
17413// FastEmit functions for X86ISD::PACKUS.
17414
17415Register fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17416 if (RetVT.SimpleTy != MVT::v16i8)
17417 return Register();
17418 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17419 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17420 }
17421 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17422 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17423 }
17424 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17425 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17426 }
17427 return Register();
17428}
17429
17430Register fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17431 if (RetVT.SimpleTy != MVT::v32i8)
17432 return Register();
17433 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17434 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17435 }
17436 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17437 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17438 }
17439 return Register();
17440}
17441
17442Register fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17443 if (RetVT.SimpleTy != MVT::v64i8)
17444 return Register();
17445 if ((Subtarget->hasBWI())) {
17446 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17447 }
17448 return Register();
17449}
17450
17451Register fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17452 if (RetVT.SimpleTy != MVT::v8i16)
17453 return Register();
17454 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17455 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17456 }
17457 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17458 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17459 }
17460 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17461 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17462 }
17463 return Register();
17464}
17465
17466Register fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17467 if (RetVT.SimpleTy != MVT::v16i16)
17468 return Register();
17469 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17470 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17471 }
17472 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17473 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17474 }
17475 return Register();
17476}
17477
17478Register fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
17479 if (RetVT.SimpleTy != MVT::v32i16)
17480 return Register();
17481 if ((Subtarget->hasBWI())) {
17482 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17483 }
17484 return Register();
17485}
17486
17487Register fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17488 switch (VT.SimpleTy) {
17489 case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op1);
17490 case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op1);
17491 case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op1);
17492 case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op1);
17493 case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op1);
17494 case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op1);
17495 default: return Register();
17496 }
17497}
17498
17499// FastEmit functions for X86ISD::PCMPEQ.
17500
17501Register fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17502 if (RetVT.SimpleTy != MVT::v16i8)
17503 return Register();
17504 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17505 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
17506 }
17507 if ((Subtarget->hasAVX()) && (true)) {
17508 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
17509 }
17510 return Register();
17511}
17512
17513Register fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17514 if (RetVT.SimpleTy != MVT::v32i8)
17515 return Register();
17516 if ((Subtarget->hasAVX2()) && (true)) {
17517 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17518 }
17519 return Register();
17520}
17521
17522Register fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17523 if (RetVT.SimpleTy != MVT::v8i16)
17524 return Register();
17525 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17526 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
17527 }
17528 if ((Subtarget->hasAVX()) && (true)) {
17529 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
17530 }
17531 return Register();
17532}
17533
17534Register fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17535 if (RetVT.SimpleTy != MVT::v16i16)
17536 return Register();
17537 if ((Subtarget->hasAVX2()) && (true)) {
17538 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17539 }
17540 return Register();
17541}
17542
17543Register fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17544 if (RetVT.SimpleTy != MVT::v4i32)
17545 return Register();
17546 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17547 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
17548 }
17549 if ((Subtarget->hasAVX()) && (true)) {
17550 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
17551 }
17552 return Register();
17553}
17554
17555Register fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17556 if (RetVT.SimpleTy != MVT::v8i32)
17557 return Register();
17558 if ((Subtarget->hasAVX2()) && (true)) {
17559 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17560 }
17561 return Register();
17562}
17563
17564Register fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17565 if (RetVT.SimpleTy != MVT::v2i64)
17566 return Register();
17567 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17568 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
17569 }
17570 if ((Subtarget->hasAVX())) {
17571 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
17572 }
17573 return Register();
17574}
17575
17576Register fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17577 if (RetVT.SimpleTy != MVT::v4i64)
17578 return Register();
17579 if ((Subtarget->hasAVX2())) {
17580 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17581 }
17582 return Register();
17583}
17584
17585Register fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17586 switch (VT.SimpleTy) {
17587 case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op1);
17588 case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op1);
17589 case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op1);
17590 case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op1);
17591 case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op1);
17592 case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op1);
17593 case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17594 case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17595 default: return Register();
17596 }
17597}
17598
17599// FastEmit functions for X86ISD::PCMPGT.
17600
17601Register fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17602 if (RetVT.SimpleTy != MVT::v16i8)
17603 return Register();
17604 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17605 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
17606 }
17607 if ((Subtarget->hasAVX()) && (true)) {
17608 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
17609 }
17610 return Register();
17611}
17612
17613Register fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17614 if (RetVT.SimpleTy != MVT::v32i8)
17615 return Register();
17616 if ((Subtarget->hasAVX2()) && (true)) {
17617 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17618 }
17619 return Register();
17620}
17621
17622Register fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17623 if (RetVT.SimpleTy != MVT::v8i16)
17624 return Register();
17625 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17626 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
17627 }
17628 if ((Subtarget->hasAVX()) && (true)) {
17629 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
17630 }
17631 return Register();
17632}
17633
17634Register fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17635 if (RetVT.SimpleTy != MVT::v16i16)
17636 return Register();
17637 if ((Subtarget->hasAVX2()) && (true)) {
17638 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17639 }
17640 return Register();
17641}
17642
17643Register fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17644 if (RetVT.SimpleTy != MVT::v4i32)
17645 return Register();
17646 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17647 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
17648 }
17649 if ((Subtarget->hasAVX()) && (true)) {
17650 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
17651 }
17652 return Register();
17653}
17654
17655Register fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17656 if (RetVT.SimpleTy != MVT::v8i32)
17657 return Register();
17658 if ((Subtarget->hasAVX2()) && (true)) {
17659 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17660 }
17661 return Register();
17662}
17663
17664Register fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17665 if (RetVT.SimpleTy != MVT::v2i64)
17666 return Register();
17667 if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) {
17668 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
17669 }
17670 if ((Subtarget->hasAVX())) {
17671 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
17672 }
17673 return Register();
17674}
17675
17676Register fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17677 if (RetVT.SimpleTy != MVT::v4i64)
17678 return Register();
17679 if ((Subtarget->hasAVX2())) {
17680 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17681 }
17682 return Register();
17683}
17684
17685Register fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17686 switch (VT.SimpleTy) {
17687 case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op1);
17688 case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op1);
17689 case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op1);
17690 case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op1);
17691 case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op1);
17692 case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op1);
17693 case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op1);
17694 case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op1);
17695 default: return Register();
17696 }
17697}
17698
17699// FastEmit functions for X86ISD::PMULDQ.
17700
17701Register fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17702 if (RetVT.SimpleTy != MVT::v2i64)
17703 return Register();
17704 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17705 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17706 }
17707 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17708 return fastEmitInst_rr(MachineInstOpcode: X86::PMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17709 }
17710 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17711 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17712 }
17713 return Register();
17714}
17715
17716Register fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17717 if (RetVT.SimpleTy != MVT::v4i64)
17718 return Register();
17719 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17720 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17721 }
17722 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17723 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17724 }
17725 return Register();
17726}
17727
17728Register fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
17729 if (RetVT.SimpleTy != MVT::v8i64)
17730 return Register();
17731 if ((Subtarget->hasAVX512())) {
17732 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17733 }
17734 return Register();
17735}
17736
17737Register fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17738 switch (VT.SimpleTy) {
17739 case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17740 case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17741 case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
17742 default: return Register();
17743 }
17744}
17745
17746// FastEmit functions for X86ISD::PMULUDQ.
17747
17748Register fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17749 if (RetVT.SimpleTy != MVT::v2i64)
17750 return Register();
17751 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17752 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17753 }
17754 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17755 return fastEmitInst_rr(MachineInstOpcode: X86::PMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17756 }
17757 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17758 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17759 }
17760 return Register();
17761}
17762
17763Register fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17764 if (RetVT.SimpleTy != MVT::v4i64)
17765 return Register();
17766 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17767 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17768 }
17769 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17770 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17771 }
17772 return Register();
17773}
17774
17775Register fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
17776 if (RetVT.SimpleTy != MVT::v8i64)
17777 return Register();
17778 if ((Subtarget->hasAVX512())) {
17779 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17780 }
17781 return Register();
17782}
17783
17784Register fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17785 switch (VT.SimpleTy) {
17786 case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17787 case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17788 case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
17789 default: return Register();
17790 }
17791}
17792
17793// FastEmit functions for X86ISD::PSADBW.
17794
17795Register fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17796 if (RetVT.SimpleTy != MVT::v2i64)
17797 return Register();
17798 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17799 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17800 }
17801 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17802 return fastEmitInst_rr(MachineInstOpcode: X86::PSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17803 }
17804 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17805 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17806 }
17807 return Register();
17808}
17809
17810Register fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17811 if (RetVT.SimpleTy != MVT::v4i64)
17812 return Register();
17813 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17814 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17815 }
17816 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17817 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17818 }
17819 return Register();
17820}
17821
17822Register fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17823 if (RetVT.SimpleTy != MVT::v8i64)
17824 return Register();
17825 if ((Subtarget->hasBWI())) {
17826 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17827 }
17828 return Register();
17829}
17830
17831Register fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17832 switch (VT.SimpleTy) {
17833 case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op1);
17834 case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op1);
17835 case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op1);
17836 default: return Register();
17837 }
17838}
17839
17840// FastEmit functions for X86ISD::PSHUFB.
17841
17842Register fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17843 if (RetVT.SimpleTy != MVT::v16i8)
17844 return Register();
17845 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17846 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17847 }
17848 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17849 return fastEmitInst_rr(MachineInstOpcode: X86::PSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
17850 }
17851 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17852 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
17853 }
17854 return Register();
17855}
17856
17857Register fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17858 if (RetVT.SimpleTy != MVT::v32i8)
17859 return Register();
17860 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17861 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17862 }
17863 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17864 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17865 }
17866 return Register();
17867}
17868
17869Register fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17870 if (RetVT.SimpleTy != MVT::v64i8)
17871 return Register();
17872 if ((Subtarget->hasBWI())) {
17873 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17874 }
17875 return Register();
17876}
17877
17878Register fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17879 switch (VT.SimpleTy) {
17880 case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op1);
17881 case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op1);
17882 case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op1);
17883 default: return Register();
17884 }
17885}
17886
17887// FastEmit functions for X86ISD::PTEST.
17888
17889Register fastEmit_X86ISD_PTEST_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17890 if (RetVT.SimpleTy != MVT::i32)
17891 return Register();
17892 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17893 return fastEmitInst_rr(MachineInstOpcode: X86::PTESTrr, RC: &X86::VR128RegClass, Op0, Op1);
17894 }
17895 if ((Subtarget->hasAVX())) {
17896 return fastEmitInst_rr(MachineInstOpcode: X86::VPTESTrr, RC: &X86::VR128RegClass, Op0, Op1);
17897 }
17898 return Register();
17899}
17900
17901Register fastEmit_X86ISD_PTEST_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17902 if (RetVT.SimpleTy != MVT::i32)
17903 return Register();
17904 if ((Subtarget->hasAVX())) {
17905 return fastEmitInst_rr(MachineInstOpcode: X86::VPTESTYrr, RC: &X86::VR256RegClass, Op0, Op1);
17906 }
17907 return Register();
17908}
17909
17910Register fastEmit_X86ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17911 switch (VT.SimpleTy) {
17912 case MVT::v2i64: return fastEmit_X86ISD_PTEST_MVT_v2i64_rr(RetVT, Op0, Op1);
17913 case MVT::v4i64: return fastEmit_X86ISD_PTEST_MVT_v4i64_rr(RetVT, Op0, Op1);
17914 default: return Register();
17915 }
17916}
17917
17918// FastEmit functions for X86ISD::RCP14S.
17919
17920Register fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17921 if (RetVT.SimpleTy != MVT::v8f16)
17922 return Register();
17923 if ((Subtarget->hasFP16())) {
17924 return fastEmitInst_rr(MachineInstOpcode: X86::VRCPSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17925 }
17926 return Register();
17927}
17928
17929Register fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17930 if (RetVT.SimpleTy != MVT::v4f32)
17931 return Register();
17932 if ((Subtarget->hasAVX512())) {
17933 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17934 }
17935 return Register();
17936}
17937
17938Register fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17939 if (RetVT.SimpleTy != MVT::v2f64)
17940 return Register();
17941 if ((Subtarget->hasAVX512())) {
17942 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17943 }
17944 return Register();
17945}
17946
17947Register fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17948 switch (VT.SimpleTy) {
17949 case MVT::v8f16: return fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(RetVT, Op0, Op1);
17950 case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op1);
17951 case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op1);
17952 default: return Register();
17953 }
17954}
17955
17956// FastEmit functions for X86ISD::RSQRT14S.
17957
17958Register fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17959 if (RetVT.SimpleTy != MVT::v8f16)
17960 return Register();
17961 if ((Subtarget->hasFP16())) {
17962 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRTSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17963 }
17964 return Register();
17965}
17966
17967Register fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17968 if (RetVT.SimpleTy != MVT::v4f32)
17969 return Register();
17970 if ((Subtarget->hasAVX512())) {
17971 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17972 }
17973 return Register();
17974}
17975
17976Register fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17977 if (RetVT.SimpleTy != MVT::v2f64)
17978 return Register();
17979 if ((Subtarget->hasAVX512())) {
17980 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17981 }
17982 return Register();
17983}
17984
17985Register fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17986 switch (VT.SimpleTy) {
17987 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(RetVT, Op0, Op1);
17988 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op1);
17989 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op1);
17990 default: return Register();
17991 }
17992}
17993
17994// FastEmit functions for X86ISD::SCALEF.
17995
17996Register fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17997 if (RetVT.SimpleTy != MVT::v8f16)
17998 return Register();
17999 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18000 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18001 }
18002 return Register();
18003}
18004
18005Register fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18006 if (RetVT.SimpleTy != MVT::v16f16)
18007 return Register();
18008 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18009 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18010 }
18011 return Register();
18012}
18013
18014Register fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18015 if (RetVT.SimpleTy != MVT::v32f16)
18016 return Register();
18017 if ((Subtarget->hasFP16())) {
18018 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
18019 }
18020 return Register();
18021}
18022
18023Register fastEmit_X86ISD_SCALEF_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
18024 if (RetVT.SimpleTy != MVT::v8bf16)
18025 return Register();
18026 if ((Subtarget->hasAVX10_2())) {
18027 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18028 }
18029 return Register();
18030}
18031
18032Register fastEmit_X86ISD_SCALEF_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
18033 if (RetVT.SimpleTy != MVT::v16bf16)
18034 return Register();
18035 if ((Subtarget->hasAVX10_2())) {
18036 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18037 }
18038 return Register();
18039}
18040
18041Register fastEmit_X86ISD_SCALEF_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
18042 if (RetVT.SimpleTy != MVT::v32bf16)
18043 return Register();
18044 if ((Subtarget->hasAVX10_2())) {
18045 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
18046 }
18047 return Register();
18048}
18049
18050Register fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18051 if (RetVT.SimpleTy != MVT::v4f32)
18052 return Register();
18053 if ((Subtarget->hasVLX())) {
18054 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18055 }
18056 return Register();
18057}
18058
18059Register fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18060 if (RetVT.SimpleTy != MVT::v8f32)
18061 return Register();
18062 if ((Subtarget->hasVLX())) {
18063 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18064 }
18065 return Register();
18066}
18067
18068Register fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18069 if (RetVT.SimpleTy != MVT::v16f32)
18070 return Register();
18071 if ((Subtarget->hasAVX512())) {
18072 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18073 }
18074 return Register();
18075}
18076
18077Register fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18078 if (RetVT.SimpleTy != MVT::v2f64)
18079 return Register();
18080 if ((Subtarget->hasVLX())) {
18081 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18082 }
18083 return Register();
18084}
18085
18086Register fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18087 if (RetVT.SimpleTy != MVT::v4f64)
18088 return Register();
18089 if ((Subtarget->hasVLX())) {
18090 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18091 }
18092 return Register();
18093}
18094
18095Register fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18096 if (RetVT.SimpleTy != MVT::v8f64)
18097 return Register();
18098 if ((Subtarget->hasAVX512())) {
18099 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18100 }
18101 return Register();
18102}
18103
18104Register fastEmit_X86ISD_SCALEF_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18105 switch (VT.SimpleTy) {
18106 case MVT::v8f16: return fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(RetVT, Op0, Op1);
18107 case MVT::v16f16: return fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(RetVT, Op0, Op1);
18108 case MVT::v32f16: return fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(RetVT, Op0, Op1);
18109 case MVT::v8bf16: return fastEmit_X86ISD_SCALEF_MVT_v8bf16_rr(RetVT, Op0, Op1);
18110 case MVT::v16bf16: return fastEmit_X86ISD_SCALEF_MVT_v16bf16_rr(RetVT, Op0, Op1);
18111 case MVT::v32bf16: return fastEmit_X86ISD_SCALEF_MVT_v32bf16_rr(RetVT, Op0, Op1);
18112 case MVT::v4f32: return fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(RetVT, Op0, Op1);
18113 case MVT::v8f32: return fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(RetVT, Op0, Op1);
18114 case MVT::v16f32: return fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(RetVT, Op0, Op1);
18115 case MVT::v2f64: return fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(RetVT, Op0, Op1);
18116 case MVT::v4f64: return fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(RetVT, Op0, Op1);
18117 case MVT::v8f64: return fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(RetVT, Op0, Op1);
18118 default: return Register();
18119 }
18120}
18121
18122// FastEmit functions for X86ISD::SCALEFS.
18123
18124Register fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18125 if (RetVT.SimpleTy != MVT::v8f16)
18126 return Register();
18127 if ((Subtarget->hasFP16())) {
18128 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18129 }
18130 return Register();
18131}
18132
18133Register fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18134 if (RetVT.SimpleTy != MVT::v4f32)
18135 return Register();
18136 if ((Subtarget->hasAVX512())) {
18137 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18138 }
18139 return Register();
18140}
18141
18142Register fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18143 if (RetVT.SimpleTy != MVT::v2f64)
18144 return Register();
18145 if ((Subtarget->hasAVX512())) {
18146 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18147 }
18148 return Register();
18149}
18150
18151Register fastEmit_X86ISD_SCALEFS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18152 switch (VT.SimpleTy) {
18153 case MVT::v8f16: return fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(RetVT, Op0, Op1);
18154 case MVT::v4f32: return fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(RetVT, Op0, Op1);
18155 case MVT::v2f64: return fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(RetVT, Op0, Op1);
18156 default: return Register();
18157 }
18158}
18159
18160// FastEmit functions for X86ISD::STRICT_FCMP.
18161
18162Register fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18163 if (RetVT.SimpleTy != MVT::i32)
18164 return Register();
18165 if ((Subtarget->hasFP16())) {
18166 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18167 }
18168 return Register();
18169}
18170
18171Register fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18172 if (RetVT.SimpleTy != MVT::i32)
18173 return Register();
18174 if ((Subtarget->hasAVX512())) {
18175 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18176 }
18177 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18178 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18179 }
18180 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18181 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18182 }
18183 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
18184 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
18185 }
18186 return Register();
18187}
18188
18189Register fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18190 if (RetVT.SimpleTy != MVT::i32)
18191 return Register();
18192 if ((Subtarget->hasAVX512())) {
18193 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18194 }
18195 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18196 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18197 }
18198 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18199 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18200 }
18201 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
18202 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
18203 }
18204 return Register();
18205}
18206
18207Register fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18208 if (RetVT.SimpleTy != MVT::i32)
18209 return Register();
18210 if ((Subtarget->canUseCMOV())) {
18211 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
18212 }
18213 return Register();
18214}
18215
18216Register fastEmit_X86ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18217 switch (VT.SimpleTy) {
18218 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
18219 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
18220 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
18221 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
18222 default: return Register();
18223 }
18224}
18225
18226// FastEmit functions for X86ISD::STRICT_FCMPS.
18227
18228Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18229 if (RetVT.SimpleTy != MVT::i32)
18230 return Register();
18231 if ((Subtarget->hasFP16())) {
18232 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18233 }
18234 return Register();
18235}
18236
18237Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18238 if (RetVT.SimpleTy != MVT::i32)
18239 return Register();
18240 if ((Subtarget->hasAVX512())) {
18241 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18242 }
18243 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18244 return fastEmitInst_rr(MachineInstOpcode: X86::COMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18245 }
18246 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18247 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18248 }
18249 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
18250 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
18251 }
18252 return Register();
18253}
18254
18255Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18256 if (RetVT.SimpleTy != MVT::i32)
18257 return Register();
18258 if ((Subtarget->hasAVX512())) {
18259 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18260 }
18261 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18262 return fastEmitInst_rr(MachineInstOpcode: X86::COMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18263 }
18264 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18265 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18266 }
18267 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
18268 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
18269 }
18270 return Register();
18271}
18272
18273Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18274 if (RetVT.SimpleTy != MVT::i32)
18275 return Register();
18276 if ((Subtarget->canUseCMOV())) {
18277 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
18278 }
18279 return Register();
18280}
18281
18282Register fastEmit_X86ISD_STRICT_FCMPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18283 switch (VT.SimpleTy) {
18284 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(RetVT, Op0, Op1);
18285 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(RetVT, Op0, Op1);
18286 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(RetVT, Op0, Op1);
18287 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(RetVT, Op0, Op1);
18288 default: return Register();
18289 }
18290}
18291
18292// FastEmit functions for X86ISD::STRICT_FMAX.
18293
18294Register fastEmit_X86ISD_STRICT_FMAX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18295 if (RetVT.SimpleTy != MVT::f16)
18296 return Register();
18297 if ((Subtarget->hasFP16())) {
18298 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18299 }
18300 return Register();
18301}
18302
18303Register fastEmit_X86ISD_STRICT_FMAX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18304 if (RetVT.SimpleTy != MVT::f32)
18305 return Register();
18306 if ((Subtarget->hasAVX512())) {
18307 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18308 }
18309 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18310 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18311 }
18312 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18313 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18314 }
18315 return Register();
18316}
18317
18318Register fastEmit_X86ISD_STRICT_FMAX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18319 if (RetVT.SimpleTy != MVT::f64)
18320 return Register();
18321 if ((Subtarget->hasAVX512())) {
18322 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18323 }
18324 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18325 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18326 }
18327 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18328 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18329 }
18330 return Register();
18331}
18332
18333Register fastEmit_X86ISD_STRICT_FMAX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18334 if (RetVT.SimpleTy != MVT::v8f16)
18335 return Register();
18336 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18337 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18338 }
18339 return Register();
18340}
18341
18342Register fastEmit_X86ISD_STRICT_FMAX_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18343 if (RetVT.SimpleTy != MVT::v16f16)
18344 return Register();
18345 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18346 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18347 }
18348 return Register();
18349}
18350
18351Register fastEmit_X86ISD_STRICT_FMAX_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18352 if (RetVT.SimpleTy != MVT::v32f16)
18353 return Register();
18354 if ((Subtarget->hasFP16())) {
18355 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
18356 }
18357 return Register();
18358}
18359
18360Register fastEmit_X86ISD_STRICT_FMAX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18361 if (RetVT.SimpleTy != MVT::v4f32)
18362 return Register();
18363 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18364 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18365 }
18366 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18367 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18368 }
18369 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18370 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18371 }
18372 return Register();
18373}
18374
18375Register fastEmit_X86ISD_STRICT_FMAX_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18376 if (RetVT.SimpleTy != MVT::v8f32)
18377 return Register();
18378 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18379 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18380 }
18381 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18382 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18383 }
18384 return Register();
18385}
18386
18387Register fastEmit_X86ISD_STRICT_FMAX_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18388 if (RetVT.SimpleTy != MVT::v16f32)
18389 return Register();
18390 if ((Subtarget->hasAVX512())) {
18391 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18392 }
18393 return Register();
18394}
18395
18396Register fastEmit_X86ISD_STRICT_FMAX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18397 if (RetVT.SimpleTy != MVT::v2f64)
18398 return Register();
18399 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18400 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18401 }
18402 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18403 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18404 }
18405 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18406 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18407 }
18408 return Register();
18409}
18410
18411Register fastEmit_X86ISD_STRICT_FMAX_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18412 if (RetVT.SimpleTy != MVT::v4f64)
18413 return Register();
18414 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18415 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18416 }
18417 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18418 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18419 }
18420 return Register();
18421}
18422
18423Register fastEmit_X86ISD_STRICT_FMAX_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18424 if (RetVT.SimpleTy != MVT::v8f64)
18425 return Register();
18426 if ((Subtarget->hasAVX512())) {
18427 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18428 }
18429 return Register();
18430}
18431
18432Register fastEmit_X86ISD_STRICT_FMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18433 switch (VT.SimpleTy) {
18434 case MVT::f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
18435 case MVT::f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
18436 case MVT::f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
18437 case MVT::v8f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
18438 case MVT::v16f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
18439 case MVT::v32f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
18440 case MVT::v4f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
18441 case MVT::v8f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
18442 case MVT::v16f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
18443 case MVT::v2f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
18444 case MVT::v4f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
18445 case MVT::v8f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
18446 default: return Register();
18447 }
18448}
18449
18450// FastEmit functions for X86ISD::STRICT_FMIN.
18451
18452Register fastEmit_X86ISD_STRICT_FMIN_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18453 if (RetVT.SimpleTy != MVT::f16)
18454 return Register();
18455 if ((Subtarget->hasFP16())) {
18456 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18457 }
18458 return Register();
18459}
18460
18461Register fastEmit_X86ISD_STRICT_FMIN_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18462 if (RetVT.SimpleTy != MVT::f32)
18463 return Register();
18464 if ((Subtarget->hasAVX512())) {
18465 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18466 }
18467 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18468 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18469 }
18470 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18471 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18472 }
18473 return Register();
18474}
18475
18476Register fastEmit_X86ISD_STRICT_FMIN_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18477 if (RetVT.SimpleTy != MVT::f64)
18478 return Register();
18479 if ((Subtarget->hasAVX512())) {
18480 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18481 }
18482 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18483 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18484 }
18485 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18486 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18487 }
18488 return Register();
18489}
18490
18491Register fastEmit_X86ISD_STRICT_FMIN_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18492 if (RetVT.SimpleTy != MVT::v8f16)
18493 return Register();
18494 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18495 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18496 }
18497 return Register();
18498}
18499
18500Register fastEmit_X86ISD_STRICT_FMIN_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18501 if (RetVT.SimpleTy != MVT::v16f16)
18502 return Register();
18503 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18504 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18505 }
18506 return Register();
18507}
18508
18509Register fastEmit_X86ISD_STRICT_FMIN_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18510 if (RetVT.SimpleTy != MVT::v32f16)
18511 return Register();
18512 if ((Subtarget->hasFP16())) {
18513 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
18514 }
18515 return Register();
18516}
18517
18518Register fastEmit_X86ISD_STRICT_FMIN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18519 if (RetVT.SimpleTy != MVT::v4f32)
18520 return Register();
18521 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18522 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18523 }
18524 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18525 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18526 }
18527 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18528 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18529 }
18530 return Register();
18531}
18532
18533Register fastEmit_X86ISD_STRICT_FMIN_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18534 if (RetVT.SimpleTy != MVT::v8f32)
18535 return Register();
18536 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18537 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18538 }
18539 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18540 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18541 }
18542 return Register();
18543}
18544
18545Register fastEmit_X86ISD_STRICT_FMIN_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18546 if (RetVT.SimpleTy != MVT::v16f32)
18547 return Register();
18548 if ((Subtarget->hasAVX512())) {
18549 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18550 }
18551 return Register();
18552}
18553
18554Register fastEmit_X86ISD_STRICT_FMIN_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18555 if (RetVT.SimpleTy != MVT::v2f64)
18556 return Register();
18557 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18558 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18559 }
18560 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18561 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18562 }
18563 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18564 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18565 }
18566 return Register();
18567}
18568
18569Register fastEmit_X86ISD_STRICT_FMIN_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18570 if (RetVT.SimpleTy != MVT::v4f64)
18571 return Register();
18572 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18573 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18574 }
18575 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18576 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18577 }
18578 return Register();
18579}
18580
18581Register fastEmit_X86ISD_STRICT_FMIN_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18582 if (RetVT.SimpleTy != MVT::v8f64)
18583 return Register();
18584 if ((Subtarget->hasAVX512())) {
18585 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18586 }
18587 return Register();
18588}
18589
18590Register fastEmit_X86ISD_STRICT_FMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18591 switch (VT.SimpleTy) {
18592 case MVT::f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
18593 case MVT::f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
18594 case MVT::f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
18595 case MVT::v8f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
18596 case MVT::v16f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
18597 case MVT::v32f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
18598 case MVT::v4f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
18599 case MVT::v8f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
18600 case MVT::v16f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
18601 case MVT::v2f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
18602 case MVT::v4f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
18603 case MVT::v8f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
18604 default: return Register();
18605 }
18606}
18607
18608// FastEmit functions for X86ISD::STRICT_FP80_ADD.
18609
18610Register fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18611 if (RetVT.SimpleTy != MVT::f80)
18612 return Register();
18613 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
18614}
18615
18616Register fastEmit_X86ISD_STRICT_FP80_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18617 switch (VT.SimpleTy) {
18618 case MVT::f80: return fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
18619 default: return Register();
18620 }
18621}
18622
18623// FastEmit functions for X86ISD::TESTP.
18624
18625Register fastEmit_X86ISD_TESTP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18626 if (RetVT.SimpleTy != MVT::i32)
18627 return Register();
18628 if ((Subtarget->hasAVX())) {
18629 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18630 }
18631 return Register();
18632}
18633
18634Register fastEmit_X86ISD_TESTP_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18635 if (RetVT.SimpleTy != MVT::i32)
18636 return Register();
18637 if ((Subtarget->hasAVX())) {
18638 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18639 }
18640 return Register();
18641}
18642
18643Register fastEmit_X86ISD_TESTP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18644 if (RetVT.SimpleTy != MVT::i32)
18645 return Register();
18646 if ((Subtarget->hasAVX())) {
18647 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18648 }
18649 return Register();
18650}
18651
18652Register fastEmit_X86ISD_TESTP_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18653 if (RetVT.SimpleTy != MVT::i32)
18654 return Register();
18655 if ((Subtarget->hasAVX())) {
18656 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18657 }
18658 return Register();
18659}
18660
18661Register fastEmit_X86ISD_TESTP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18662 switch (VT.SimpleTy) {
18663 case MVT::v4f32: return fastEmit_X86ISD_TESTP_MVT_v4f32_rr(RetVT, Op0, Op1);
18664 case MVT::v8f32: return fastEmit_X86ISD_TESTP_MVT_v8f32_rr(RetVT, Op0, Op1);
18665 case MVT::v2f64: return fastEmit_X86ISD_TESTP_MVT_v2f64_rr(RetVT, Op0, Op1);
18666 case MVT::v4f64: return fastEmit_X86ISD_TESTP_MVT_v4f64_rr(RetVT, Op0, Op1);
18667 default: return Register();
18668 }
18669}
18670
18671// FastEmit functions for X86ISD::UCOMI.
18672
18673Register fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18674 if (RetVT.SimpleTy != MVT::i32)
18675 return Register();
18676 if ((Subtarget->hasFP16())) {
18677 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18678 }
18679 return Register();
18680}
18681
18682Register fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18683 if (RetVT.SimpleTy != MVT::i32)
18684 return Register();
18685 if ((Subtarget->hasAVX512())) {
18686 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18687 }
18688 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18689 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18690 }
18691 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18692 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18693 }
18694 return Register();
18695}
18696
18697Register fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18698 if (RetVT.SimpleTy != MVT::i32)
18699 return Register();
18700 if ((Subtarget->hasAVX512())) {
18701 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18702 }
18703 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18704 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18705 }
18706 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18707 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18708 }
18709 return Register();
18710}
18711
18712Register fastEmit_X86ISD_UCOMI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18713 switch (VT.SimpleTy) {
18714 case MVT::v8f16: return fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(RetVT, Op0, Op1);
18715 case MVT::v4f32: return fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(RetVT, Op0, Op1);
18716 case MVT::v2f64: return fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(RetVT, Op0, Op1);
18717 default: return Register();
18718 }
18719}
18720
18721// FastEmit functions for X86ISD::UCOMX.
18722
18723Register fastEmit_X86ISD_UCOMX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18724 if (RetVT.SimpleTy != MVT::i32)
18725 return Register();
18726 if ((Subtarget->hasAVX10_2())) {
18727 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18728 }
18729 return Register();
18730}
18731
18732Register fastEmit_X86ISD_UCOMX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18733 if (RetVT.SimpleTy != MVT::i32)
18734 return Register();
18735 if ((Subtarget->hasAVX10_2())) {
18736 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18737 }
18738 return Register();
18739}
18740
18741Register fastEmit_X86ISD_UCOMX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18742 if (RetVT.SimpleTy != MVT::i32)
18743 return Register();
18744 if ((Subtarget->hasAVX10_2())) {
18745 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18746 }
18747 return Register();
18748}
18749
18750Register fastEmit_X86ISD_UCOMX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18751 if (RetVT.SimpleTy != MVT::i32)
18752 return Register();
18753 if ((Subtarget->hasAVX10_2())) {
18754 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18755 }
18756 return Register();
18757}
18758
18759Register fastEmit_X86ISD_UCOMX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18760 if (RetVT.SimpleTy != MVT::i32)
18761 return Register();
18762 if ((Subtarget->hasAVX10_2())) {
18763 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18764 }
18765 return Register();
18766}
18767
18768Register fastEmit_X86ISD_UCOMX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18769 if (RetVT.SimpleTy != MVT::i32)
18770 return Register();
18771 if ((Subtarget->hasAVX10_2())) {
18772 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18773 }
18774 return Register();
18775}
18776
18777Register fastEmit_X86ISD_UCOMX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18778 switch (VT.SimpleTy) {
18779 case MVT::f16: return fastEmit_X86ISD_UCOMX_MVT_f16_rr(RetVT, Op0, Op1);
18780 case MVT::f32: return fastEmit_X86ISD_UCOMX_MVT_f32_rr(RetVT, Op0, Op1);
18781 case MVT::f64: return fastEmit_X86ISD_UCOMX_MVT_f64_rr(RetVT, Op0, Op1);
18782 case MVT::v8f16: return fastEmit_X86ISD_UCOMX_MVT_v8f16_rr(RetVT, Op0, Op1);
18783 case MVT::v4f32: return fastEmit_X86ISD_UCOMX_MVT_v4f32_rr(RetVT, Op0, Op1);
18784 case MVT::v2f64: return fastEmit_X86ISD_UCOMX_MVT_v2f64_rr(RetVT, Op0, Op1);
18785 default: return Register();
18786 }
18787}
18788
18789// FastEmit functions for X86ISD::UNPCKH.
18790
18791Register fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18792 if (RetVT.SimpleTy != MVT::v16i8)
18793 return Register();
18794 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18795 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18796 }
18797 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18798 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18799 }
18800 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18801 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18802 }
18803 return Register();
18804}
18805
18806Register fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18807 if (RetVT.SimpleTy != MVT::v32i8)
18808 return Register();
18809 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18810 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18811 }
18812 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18813 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
18814 }
18815 return Register();
18816}
18817
18818Register fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
18819 if (RetVT.SimpleTy != MVT::v64i8)
18820 return Register();
18821 if ((Subtarget->hasBWI())) {
18822 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
18823 }
18824 return Register();
18825}
18826
18827Register fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
18828 if (RetVT.SimpleTy != MVT::v8i16)
18829 return Register();
18830 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18831 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18832 }
18833 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18834 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18835 }
18836 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18837 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18838 }
18839 return Register();
18840}
18841
18842Register fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
18843 if (RetVT.SimpleTy != MVT::v16i16)
18844 return Register();
18845 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18846 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18847 }
18848 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18849 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18850 }
18851 return Register();
18852}
18853
18854Register fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
18855 if (RetVT.SimpleTy != MVT::v32i16)
18856 return Register();
18857 if ((Subtarget->hasBWI())) {
18858 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18859 }
18860 return Register();
18861}
18862
18863Register fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
18864 if (RetVT.SimpleTy != MVT::v4i32)
18865 return Register();
18866 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18867 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18868 }
18869 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18870 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18871 }
18872 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18873 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18874 }
18875 return Register();
18876}
18877
18878Register fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
18879 if (RetVT.SimpleTy != MVT::v8i32)
18880 return Register();
18881 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18882 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18883 }
18884 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18885 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18886 }
18887 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18888 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18889 }
18890 return Register();
18891}
18892
18893Register fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
18894 if (RetVT.SimpleTy != MVT::v16i32)
18895 return Register();
18896 if ((Subtarget->hasAVX512())) {
18897 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18898 }
18899 return Register();
18900}
18901
18902Register fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
18903 if (RetVT.SimpleTy != MVT::v2i64)
18904 return Register();
18905 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18906 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18907 }
18908 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18909 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18910 }
18911 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18912 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18913 }
18914 return Register();
18915}
18916
18917Register fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
18918 if (RetVT.SimpleTy != MVT::v4i64)
18919 return Register();
18920 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18921 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18922 }
18923 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18924 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18925 }
18926 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18927 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18928 }
18929 return Register();
18930}
18931
18932Register fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
18933 if (RetVT.SimpleTy != MVT::v8i64)
18934 return Register();
18935 if ((Subtarget->hasAVX512())) {
18936 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18937 }
18938 return Register();
18939}
18940
18941Register fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18942 if (RetVT.SimpleTy != MVT::v4f32)
18943 return Register();
18944 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18945 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18946 }
18947 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18948 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18949 }
18950 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18951 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18952 }
18953 return Register();
18954}
18955
18956Register fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18957 if (RetVT.SimpleTy != MVT::v8f32)
18958 return Register();
18959 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18960 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18961 }
18962 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18963 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18964 }
18965 return Register();
18966}
18967
18968Register fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18969 if (RetVT.SimpleTy != MVT::v16f32)
18970 return Register();
18971 if ((Subtarget->hasAVX512())) {
18972 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18973 }
18974 return Register();
18975}
18976
18977Register fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18978 if (RetVT.SimpleTy != MVT::v2f64)
18979 return Register();
18980 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18981 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18982 }
18983 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18984 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18985 }
18986 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18987 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18988 }
18989 return Register();
18990}
18991
18992Register fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18993 if (RetVT.SimpleTy != MVT::v4f64)
18994 return Register();
18995 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18996 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18997 }
18998 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18999 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19000 }
19001 return Register();
19002}
19003
19004Register fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
19005 if (RetVT.SimpleTy != MVT::v8f64)
19006 return Register();
19007 if ((Subtarget->hasAVX512())) {
19008 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19009 }
19010 return Register();
19011}
19012
19013Register fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19014 switch (VT.SimpleTy) {
19015 case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op1);
19016 case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op1);
19017 case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op1);
19018 case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op1);
19019 case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op1);
19020 case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op1);
19021 case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op1);
19022 case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op1);
19023 case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op1);
19024 case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op1);
19025 case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op1);
19026 case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op1);
19027 case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op1);
19028 case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op1);
19029 case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op1);
19030 case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op1);
19031 case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op1);
19032 case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op1);
19033 default: return Register();
19034 }
19035}
19036
19037// FastEmit functions for X86ISD::UNPCKL.
19038
19039Register fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19040 if (RetVT.SimpleTy != MVT::v16i8)
19041 return Register();
19042 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19043 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19044 }
19045 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19046 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
19047 }
19048 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19049 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
19050 }
19051 return Register();
19052}
19053
19054Register fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19055 if (RetVT.SimpleTy != MVT::v32i8)
19056 return Register();
19057 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19058 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19059 }
19060 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19061 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
19062 }
19063 return Register();
19064}
19065
19066Register fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19067 if (RetVT.SimpleTy != MVT::v64i8)
19068 return Register();
19069 if ((Subtarget->hasBWI())) {
19070 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19071 }
19072 return Register();
19073}
19074
19075Register fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19076 if (RetVT.SimpleTy != MVT::v8i16)
19077 return Register();
19078 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19079 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19080 }
19081 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19082 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19083 }
19084 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19085 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19086 }
19087 return Register();
19088}
19089
19090Register fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19091 if (RetVT.SimpleTy != MVT::v16i16)
19092 return Register();
19093 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19094 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19095 }
19096 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19097 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19098 }
19099 return Register();
19100}
19101
19102Register fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19103 if (RetVT.SimpleTy != MVT::v32i16)
19104 return Register();
19105 if ((Subtarget->hasBWI())) {
19106 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19107 }
19108 return Register();
19109}
19110
19111Register fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19112 if (RetVT.SimpleTy != MVT::v4i32)
19113 return Register();
19114 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19115 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19116 }
19117 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19118 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19119 }
19120 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19121 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19122 }
19123 return Register();
19124}
19125
19126Register fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19127 if (RetVT.SimpleTy != MVT::v8i32)
19128 return Register();
19129 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
19130 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
19131 }
19132 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19133 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19134 }
19135 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19136 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
19137 }
19138 return Register();
19139}
19140
19141Register fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19142 if (RetVT.SimpleTy != MVT::v16i32)
19143 return Register();
19144 if ((Subtarget->hasAVX512())) {
19145 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19146 }
19147 return Register();
19148}
19149
19150Register fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19151 if (RetVT.SimpleTy != MVT::v2i64)
19152 return Register();
19153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19154 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19155 }
19156 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19157 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19158 }
19159 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19160 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19161 }
19162 return Register();
19163}
19164
19165Register fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19166 if (RetVT.SimpleTy != MVT::v4i64)
19167 return Register();
19168 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
19169 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19170 }
19171 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19172 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19173 }
19174 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19175 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
19176 }
19177 return Register();
19178}
19179
19180Register fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19181 if (RetVT.SimpleTy != MVT::v8i64)
19182 return Register();
19183 if ((Subtarget->hasAVX512())) {
19184 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19185 }
19186 return Register();
19187}
19188
19189Register fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19190 if (RetVT.SimpleTy != MVT::v4f32)
19191 return Register();
19192 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19193 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19194 }
19195 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
19196 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
19197 }
19198 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19199 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
19200 }
19201 return Register();
19202}
19203
19204Register fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19205 if (RetVT.SimpleTy != MVT::v8f32)
19206 return Register();
19207 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19208 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19209 }
19210 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19211 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
19212 }
19213 return Register();
19214}
19215
19216Register fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19217 if (RetVT.SimpleTy != MVT::v16f32)
19218 return Register();
19219 if ((Subtarget->hasAVX512())) {
19220 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
19221 }
19222 return Register();
19223}
19224
19225Register fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
19226 if (RetVT.SimpleTy != MVT::v2f64)
19227 return Register();
19228 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19229 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19230 }
19231 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19232 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
19233 }
19234 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19235 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
19236 }
19237 return Register();
19238}
19239
19240Register fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
19241 if (RetVT.SimpleTy != MVT::v4f64)
19242 return Register();
19243 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19244 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19245 }
19246 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19247 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19248 }
19249 return Register();
19250}
19251
19252Register fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
19253 if (RetVT.SimpleTy != MVT::v8f64)
19254 return Register();
19255 if ((Subtarget->hasAVX512())) {
19256 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19257 }
19258 return Register();
19259}
19260
19261Register fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19262 switch (VT.SimpleTy) {
19263 case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op1);
19264 case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op1);
19265 case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op1);
19266 case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op1);
19267 case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op1);
19268 case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op1);
19269 case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op1);
19270 case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op1);
19271 case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op1);
19272 case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op1);
19273 case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op1);
19274 case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op1);
19275 case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op1);
19276 case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op1);
19277 case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op1);
19278 case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op1);
19279 case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op1);
19280 case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op1);
19281 default: return Register();
19282 }
19283}
19284
19285// FastEmit functions for X86ISD::VCVT2PH2BF8.
19286
19287Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19288 if (RetVT.SimpleTy != MVT::v16i8)
19289 return Register();
19290 if ((Subtarget->hasAVX10_2())) {
19291 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19292 }
19293 return Register();
19294}
19295
19296Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19297 if (RetVT.SimpleTy != MVT::v32i8)
19298 return Register();
19299 if ((Subtarget->hasAVX10_2())) {
19300 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19301 }
19302 return Register();
19303}
19304
19305Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19306 if (RetVT.SimpleTy != MVT::v64i8)
19307 return Register();
19308 if ((Subtarget->hasAVX10_2())) {
19309 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19310 }
19311 return Register();
19312}
19313
19314Register fastEmit_X86ISD_VCVT2PH2BF8_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19315 switch (VT.SimpleTy) {
19316 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v8f16_rr(RetVT, Op0, Op1);
19317 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v16f16_rr(RetVT, Op0, Op1);
19318 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v32f16_rr(RetVT, Op0, Op1);
19319 default: return Register();
19320 }
19321}
19322
19323// FastEmit functions for X86ISD::VCVT2PH2BF8S.
19324
19325Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19326 if (RetVT.SimpleTy != MVT::v16i8)
19327 return Register();
19328 if ((Subtarget->hasAVX10_2())) {
19329 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19330 }
19331 return Register();
19332}
19333
19334Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19335 if (RetVT.SimpleTy != MVT::v32i8)
19336 return Register();
19337 if ((Subtarget->hasAVX10_2())) {
19338 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19339 }
19340 return Register();
19341}
19342
19343Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19344 if (RetVT.SimpleTy != MVT::v64i8)
19345 return Register();
19346 if ((Subtarget->hasAVX10_2())) {
19347 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZrr, RC: &X86::VR512RegClass, Op0, Op1);
19348 }
19349 return Register();
19350}
19351
19352Register fastEmit_X86ISD_VCVT2PH2BF8S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19353 switch (VT.SimpleTy) {
19354 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v8f16_rr(RetVT, Op0, Op1);
19355 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v16f16_rr(RetVT, Op0, Op1);
19356 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v32f16_rr(RetVT, Op0, Op1);
19357 default: return Register();
19358 }
19359}
19360
19361// FastEmit functions for X86ISD::VCVT2PH2HF8.
19362
19363Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19364 if (RetVT.SimpleTy != MVT::v16i8)
19365 return Register();
19366 if ((Subtarget->hasAVX10_2())) {
19367 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19368 }
19369 return Register();
19370}
19371
19372Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19373 if (RetVT.SimpleTy != MVT::v32i8)
19374 return Register();
19375 if ((Subtarget->hasAVX10_2())) {
19376 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19377 }
19378 return Register();
19379}
19380
19381Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19382 if (RetVT.SimpleTy != MVT::v64i8)
19383 return Register();
19384 if ((Subtarget->hasAVX10_2())) {
19385 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19386 }
19387 return Register();
19388}
19389
19390Register fastEmit_X86ISD_VCVT2PH2HF8_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19391 switch (VT.SimpleTy) {
19392 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v8f16_rr(RetVT, Op0, Op1);
19393 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v16f16_rr(RetVT, Op0, Op1);
19394 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v32f16_rr(RetVT, Op0, Op1);
19395 default: return Register();
19396 }
19397}
19398
19399// FastEmit functions for X86ISD::VCVT2PH2HF8S.
19400
19401Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19402 if (RetVT.SimpleTy != MVT::v16i8)
19403 return Register();
19404 if ((Subtarget->hasAVX10_2())) {
19405 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19406 }
19407 return Register();
19408}
19409
19410Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19411 if (RetVT.SimpleTy != MVT::v32i8)
19412 return Register();
19413 if ((Subtarget->hasAVX10_2())) {
19414 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19415 }
19416 return Register();
19417}
19418
19419Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19420 if (RetVT.SimpleTy != MVT::v64i8)
19421 return Register();
19422 if ((Subtarget->hasAVX10_2())) {
19423 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZrr, RC: &X86::VR512RegClass, Op0, Op1);
19424 }
19425 return Register();
19426}
19427
19428Register fastEmit_X86ISD_VCVT2PH2HF8S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19429 switch (VT.SimpleTy) {
19430 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v8f16_rr(RetVT, Op0, Op1);
19431 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v16f16_rr(RetVT, Op0, Op1);
19432 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v32f16_rr(RetVT, Op0, Op1);
19433 default: return Register();
19434 }
19435}
19436
19437// FastEmit functions for X86ISD::VFCMULC.
19438
19439Register fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19440 if (RetVT.SimpleTy != MVT::v4f32)
19441 return Register();
19442 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19443 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19444 }
19445 return Register();
19446}
19447
19448Register fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19449 if (RetVT.SimpleTy != MVT::v8f32)
19450 return Register();
19451 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19452 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19453 }
19454 return Register();
19455}
19456
19457Register fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19458 if (RetVT.SimpleTy != MVT::v16f32)
19459 return Register();
19460 if ((Subtarget->hasFP16())) {
19461 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
19462 }
19463 return Register();
19464}
19465
19466Register fastEmit_X86ISD_VFCMULC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19467 switch (VT.SimpleTy) {
19468 case MVT::v4f32: return fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
19469 case MVT::v8f32: return fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
19470 case MVT::v16f32: return fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
19471 default: return Register();
19472 }
19473}
19474
19475// FastEmit functions for X86ISD::VFCMULCSH.
19476
19477Register fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19478 if (RetVT.SimpleTy != MVT::v4f32)
19479 return Register();
19480 if ((Subtarget->hasFP16())) {
19481 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
19482 }
19483 return Register();
19484}
19485
19486Register fastEmit_X86ISD_VFCMULCSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19487 switch (VT.SimpleTy) {
19488 case MVT::v4f32: return fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
19489 default: return Register();
19490 }
19491}
19492
19493// FastEmit functions for X86ISD::VFMULC.
19494
19495Register fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19496 if (RetVT.SimpleTy != MVT::v4f32)
19497 return Register();
19498 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19499 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19500 }
19501 return Register();
19502}
19503
19504Register fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19505 if (RetVT.SimpleTy != MVT::v8f32)
19506 return Register();
19507 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19508 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19509 }
19510 return Register();
19511}
19512
19513Register fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19514 if (RetVT.SimpleTy != MVT::v16f32)
19515 return Register();
19516 if ((Subtarget->hasFP16())) {
19517 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
19518 }
19519 return Register();
19520}
19521
19522Register fastEmit_X86ISD_VFMULC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19523 switch (VT.SimpleTy) {
19524 case MVT::v4f32: return fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
19525 case MVT::v8f32: return fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
19526 case MVT::v16f32: return fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
19527 default: return Register();
19528 }
19529}
19530
19531// FastEmit functions for X86ISD::VFMULCSH.
19532
19533Register fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19534 if (RetVT.SimpleTy != MVT::v4f32)
19535 return Register();
19536 if ((Subtarget->hasFP16())) {
19537 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
19538 }
19539 return Register();
19540}
19541
19542Register fastEmit_X86ISD_VFMULCSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19543 switch (VT.SimpleTy) {
19544 case MVT::v4f32: return fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
19545 default: return Register();
19546 }
19547}
19548
19549// FastEmit functions for X86ISD::VFPROUND2.
19550
19551Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8f16_rr(Register Op0, Register Op1) {
19552 if ((Subtarget->hasAVX10_2())) {
19553 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19554 }
19555 return Register();
19556}
19557
19558Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8bf16_rr(Register Op0, Register Op1) {
19559 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
19560 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19561 }
19562 return Register();
19563}
19564
19565Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19566switch (RetVT.SimpleTy) {
19567 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8f16_rr(Op0, Op1);
19568 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8bf16_rr(Op0, Op1);
19569 default: return Register();
19570}
19571}
19572
19573Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16f16_rr(Register Op0, Register Op1) {
19574 if ((Subtarget->hasAVX10_2())) {
19575 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19576 }
19577 return Register();
19578}
19579
19580Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16bf16_rr(Register Op0, Register Op1) {
19581 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
19582 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19583 }
19584 return Register();
19585}
19586
19587Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19588switch (RetVT.SimpleTy) {
19589 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16f16_rr(Op0, Op1);
19590 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16bf16_rr(Op0, Op1);
19591 default: return Register();
19592}
19593}
19594
19595Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32f16_rr(Register Op0, Register Op1) {
19596 if ((Subtarget->hasAVX10_2())) {
19597 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZrr, RC: &X86::VR512RegClass, Op0, Op1);
19598 }
19599 return Register();
19600}
19601
19602Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32bf16_rr(Register Op0, Register Op1) {
19603 if ((Subtarget->hasBF16())) {
19604 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19605 }
19606 return Register();
19607}
19608
19609Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19610switch (RetVT.SimpleTy) {
19611 case MVT::v32f16: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32f16_rr(Op0, Op1);
19612 case MVT::v32bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32bf16_rr(Op0, Op1);
19613 default: return Register();
19614}
19615}
19616
19617Register fastEmit_X86ISD_VFPROUND2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19618 switch (VT.SimpleTy) {
19619 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_rr(RetVT, Op0, Op1);
19620 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_rr(RetVT, Op0, Op1);
19621 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_rr(RetVT, Op0, Op1);
19622 default: return Register();
19623 }
19624}
19625
19626// FastEmit functions for X86ISD::VP2INTERSECT.
19627
19628Register fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19629 if (RetVT.SimpleTy != MVT::Untyped)
19630 return Register();
19631 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19632 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ128rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
19633 }
19634 return Register();
19635}
19636
19637Register fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19638 if (RetVT.SimpleTy != MVT::Untyped)
19639 return Register();
19640 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19641 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ256rr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
19642 }
19643 return Register();
19644}
19645
19646Register fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19647 if (RetVT.SimpleTy != MVT::Untyped)
19648 return Register();
19649 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
19650 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZrr, RC: &X86::VK16PAIRRegClass, Op0, Op1);
19651 }
19652 return Register();
19653}
19654
19655Register fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19656 if (RetVT.SimpleTy != MVT::Untyped)
19657 return Register();
19658 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19659 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ128rr, RC: &X86::VK2PAIRRegClass, Op0, Op1);
19660 }
19661 return Register();
19662}
19663
19664Register fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19665 if (RetVT.SimpleTy != MVT::Untyped)
19666 return Register();
19667 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19668 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ256rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
19669 }
19670 return Register();
19671}
19672
19673Register fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19674 if (RetVT.SimpleTy != MVT::Untyped)
19675 return Register();
19676 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
19677 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZrr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
19678 }
19679 return Register();
19680}
19681
19682Register fastEmit_X86ISD_VP2INTERSECT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19683 switch (VT.SimpleTy) {
19684 case MVT::v4i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(RetVT, Op0, Op1);
19685 case MVT::v8i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(RetVT, Op0, Op1);
19686 case MVT::v16i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(RetVT, Op0, Op1);
19687 case MVT::v2i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(RetVT, Op0, Op1);
19688 case MVT::v4i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(RetVT, Op0, Op1);
19689 case MVT::v8i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(RetVT, Op0, Op1);
19690 default: return Register();
19691 }
19692}
19693
19694// FastEmit functions for X86ISD::VPERMV.
19695
19696Register fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19697 if (RetVT.SimpleTy != MVT::v16i8)
19698 return Register();
19699 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
19700 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19701 }
19702 return Register();
19703}
19704
19705Register fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19706 if (RetVT.SimpleTy != MVT::v32i8)
19707 return Register();
19708 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
19709 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19710 }
19711 return Register();
19712}
19713
19714Register fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19715 if (RetVT.SimpleTy != MVT::v64i8)
19716 return Register();
19717 if ((Subtarget->hasVBMI())) {
19718 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZrr, RC: &X86::VR512RegClass, Op0, Op1);
19719 }
19720 return Register();
19721}
19722
19723Register fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19724 if (RetVT.SimpleTy != MVT::v8i16)
19725 return Register();
19726 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19727 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19728 }
19729 return Register();
19730}
19731
19732Register fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19733 if (RetVT.SimpleTy != MVT::v16i16)
19734 return Register();
19735 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19736 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19737 }
19738 return Register();
19739}
19740
19741Register fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19742 if (RetVT.SimpleTy != MVT::v32i16)
19743 return Register();
19744 if ((Subtarget->hasBWI())) {
19745 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19746 }
19747 return Register();
19748}
19749
19750Register fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19751 if (RetVT.SimpleTy != MVT::v8i32)
19752 return Register();
19753 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19754 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19755 }
19756 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19757 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19758 }
19759 return Register();
19760}
19761
19762Register fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19763 if (RetVT.SimpleTy != MVT::v16i32)
19764 return Register();
19765 if ((Subtarget->hasAVX512())) {
19766 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19767 }
19768 return Register();
19769}
19770
19771Register fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19772 if (RetVT.SimpleTy != MVT::v4i64)
19773 return Register();
19774 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19775 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19776 }
19777 return Register();
19778}
19779
19780Register fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19781 if (RetVT.SimpleTy != MVT::v8i64)
19782 return Register();
19783 if ((Subtarget->hasAVX512())) {
19784 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19785 }
19786 return Register();
19787}
19788
19789Register fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19790 switch (VT.SimpleTy) {
19791 case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op1);
19792 case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op1);
19793 case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op1);
19794 case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op1);
19795 case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op1);
19796 case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op1);
19797 case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op1);
19798 case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op1);
19799 case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op1);
19800 case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op1);
19801 default: return Register();
19802 }
19803}
19804
19805// FastEmit functions for X86ISD::VPMADDUBSW.
19806
19807Register fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19808 if (RetVT.SimpleTy != MVT::v8i16)
19809 return Register();
19810 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19811 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19812 }
19813 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
19814 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
19815 }
19816 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19817 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
19818 }
19819 return Register();
19820}
19821
19822Register fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19823 if (RetVT.SimpleTy != MVT::v16i16)
19824 return Register();
19825 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19826 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19827 }
19828 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19829 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
19830 }
19831 return Register();
19832}
19833
19834Register fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19835 if (RetVT.SimpleTy != MVT::v32i16)
19836 return Register();
19837 if ((Subtarget->hasBWI())) {
19838 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19839 }
19840 return Register();
19841}
19842
19843Register fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19844 switch (VT.SimpleTy) {
19845 case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op1);
19846 case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op1);
19847 case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op1);
19848 default: return Register();
19849 }
19850}
19851
19852// FastEmit functions for X86ISD::VPMADDWD.
19853
19854Register fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19855 if (RetVT.SimpleTy != MVT::v4i32)
19856 return Register();
19857 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19858 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19859 }
19860 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19861 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19862 }
19863 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19864 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19865 }
19866 return Register();
19867}
19868
19869Register fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19870 if (RetVT.SimpleTy != MVT::v8i32)
19871 return Register();
19872 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19873 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19874 }
19875 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19876 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19877 }
19878 return Register();
19879}
19880
19881Register fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19882 if (RetVT.SimpleTy != MVT::v16i32)
19883 return Register();
19884 if ((Subtarget->hasBWI())) {
19885 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19886 }
19887 return Register();
19888}
19889
19890Register fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19891 switch (VT.SimpleTy) {
19892 case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op1);
19893 case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op1);
19894 case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op1);
19895 default: return Register();
19896 }
19897}
19898
19899// FastEmit functions for X86ISD::VPSHA.
19900
19901Register fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19902 if (RetVT.SimpleTy != MVT::v16i8)
19903 return Register();
19904 if ((Subtarget->hasXOP())) {
19905 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHABrr, RC: &X86::VR128RegClass, Op0, Op1);
19906 }
19907 return Register();
19908}
19909
19910Register fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19911 if (RetVT.SimpleTy != MVT::v8i16)
19912 return Register();
19913 if ((Subtarget->hasXOP())) {
19914 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAWrr, RC: &X86::VR128RegClass, Op0, Op1);
19915 }
19916 return Register();
19917}
19918
19919Register fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19920 if (RetVT.SimpleTy != MVT::v4i32)
19921 return Register();
19922 if ((Subtarget->hasXOP())) {
19923 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHADrr, RC: &X86::VR128RegClass, Op0, Op1);
19924 }
19925 return Register();
19926}
19927
19928Register fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19929 if (RetVT.SimpleTy != MVT::v2i64)
19930 return Register();
19931 if ((Subtarget->hasXOP())) {
19932 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAQrr, RC: &X86::VR128RegClass, Op0, Op1);
19933 }
19934 return Register();
19935}
19936
19937Register fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19938 switch (VT.SimpleTy) {
19939 case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op1);
19940 case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op1);
19941 case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op1);
19942 case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op1);
19943 default: return Register();
19944 }
19945}
19946
19947// FastEmit functions for X86ISD::VPSHL.
19948
19949Register fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19950 if (RetVT.SimpleTy != MVT::v16i8)
19951 return Register();
19952 if ((Subtarget->hasXOP())) {
19953 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLBrr, RC: &X86::VR128RegClass, Op0, Op1);
19954 }
19955 return Register();
19956}
19957
19958Register fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19959 if (RetVT.SimpleTy != MVT::v8i16)
19960 return Register();
19961 if ((Subtarget->hasXOP())) {
19962 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19963 }
19964 return Register();
19965}
19966
19967Register fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19968 if (RetVT.SimpleTy != MVT::v4i32)
19969 return Register();
19970 if ((Subtarget->hasXOP())) {
19971 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19972 }
19973 return Register();
19974}
19975
19976Register fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19977 if (RetVT.SimpleTy != MVT::v2i64)
19978 return Register();
19979 if ((Subtarget->hasXOP())) {
19980 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLQrr, RC: &X86::VR128RegClass, Op0, Op1);
19981 }
19982 return Register();
19983}
19984
19985Register fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19986 switch (VT.SimpleTy) {
19987 case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op1);
19988 case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
19989 case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
19990 case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
19991 default: return Register();
19992 }
19993}
19994
19995// FastEmit functions for X86ISD::VPSHUFBITQMB.
19996
19997Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19998 if (RetVT.SimpleTy != MVT::v16i1)
19999 return Register();
20000 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
20001 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ128rr, RC: &X86::VK16RegClass, Op0, Op1);
20002 }
20003 return Register();
20004}
20005
20006Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
20007 if (RetVT.SimpleTy != MVT::v32i1)
20008 return Register();
20009 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
20010 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ256rr, RC: &X86::VK32RegClass, Op0, Op1);
20011 }
20012 return Register();
20013}
20014
20015Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
20016 if (RetVT.SimpleTy != MVT::v64i1)
20017 return Register();
20018 if ((Subtarget->hasBITALG())) {
20019 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZrr, RC: &X86::VK64RegClass, Op0, Op1);
20020 }
20021 return Register();
20022}
20023
20024Register fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20025 switch (VT.SimpleTy) {
20026 case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op1);
20027 case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op1);
20028 case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op1);
20029 default: return Register();
20030 }
20031}
20032
20033// FastEmit functions for X86ISD::VSHL.
20034
20035Register fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20036 if (RetVT.SimpleTy != MVT::v8i16)
20037 return Register();
20038 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20039 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20040 }
20041 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20042 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20043 }
20044 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
20045 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20046 }
20047 return Register();
20048}
20049
20050Register fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20051 if (RetVT.SimpleTy != MVT::v4i32)
20052 return Register();
20053 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20054 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20055 }
20056 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20057 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20058 }
20059 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20060 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20061 }
20062 return Register();
20063}
20064
20065Register fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20066 if (RetVT.SimpleTy != MVT::v2i64)
20067 return Register();
20068 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20069 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20070 }
20071 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20072 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20073 }
20074 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20075 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20076 }
20077 return Register();
20078}
20079
20080Register fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20081 switch (VT.SimpleTy) {
20082 case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
20083 case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
20084 case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
20085 default: return Register();
20086 }
20087}
20088
20089// FastEmit functions for X86ISD::VSHLV.
20090
20091Register fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20092 if (RetVT.SimpleTy != MVT::v8i16)
20093 return Register();
20094 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20095 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20096 }
20097 return Register();
20098}
20099
20100Register fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20101 if (RetVT.SimpleTy != MVT::v16i16)
20102 return Register();
20103 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20104 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20105 }
20106 return Register();
20107}
20108
20109Register fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20110 if (RetVT.SimpleTy != MVT::v32i16)
20111 return Register();
20112 if ((Subtarget->hasBWI())) {
20113 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20114 }
20115 return Register();
20116}
20117
20118Register fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20119 if (RetVT.SimpleTy != MVT::v4i32)
20120 return Register();
20121 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20122 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20123 }
20124 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20125 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20126 }
20127 return Register();
20128}
20129
20130Register fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20131 if (RetVT.SimpleTy != MVT::v8i32)
20132 return Register();
20133 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20134 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20135 }
20136 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20137 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20138 }
20139 return Register();
20140}
20141
20142Register fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20143 if (RetVT.SimpleTy != MVT::v16i32)
20144 return Register();
20145 if ((Subtarget->hasAVX512())) {
20146 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20147 }
20148 return Register();
20149}
20150
20151Register fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20152 if (RetVT.SimpleTy != MVT::v2i64)
20153 return Register();
20154 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20155 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20156 }
20157 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20158 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
20159 }
20160 return Register();
20161}
20162
20163Register fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20164 if (RetVT.SimpleTy != MVT::v4i64)
20165 return Register();
20166 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20167 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20168 }
20169 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20170 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
20171 }
20172 return Register();
20173}
20174
20175Register fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20176 if (RetVT.SimpleTy != MVT::v8i64)
20177 return Register();
20178 if ((Subtarget->hasAVX512())) {
20179 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20180 }
20181 return Register();
20182}
20183
20184Register fastEmit_X86ISD_VSHLV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20185 switch (VT.SimpleTy) {
20186 case MVT::v8i16: return fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(RetVT, Op0, Op1);
20187 case MVT::v16i16: return fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(RetVT, Op0, Op1);
20188 case MVT::v32i16: return fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(RetVT, Op0, Op1);
20189 case MVT::v4i32: return fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(RetVT, Op0, Op1);
20190 case MVT::v8i32: return fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(RetVT, Op0, Op1);
20191 case MVT::v16i32: return fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(RetVT, Op0, Op1);
20192 case MVT::v2i64: return fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(RetVT, Op0, Op1);
20193 case MVT::v4i64: return fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(RetVT, Op0, Op1);
20194 case MVT::v8i64: return fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(RetVT, Op0, Op1);
20195 default: return Register();
20196 }
20197}
20198
20199// FastEmit functions for X86ISD::VSRA.
20200
20201Register fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20202 if (RetVT.SimpleTy != MVT::v8i16)
20203 return Register();
20204 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20205 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20206 }
20207 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20208 return fastEmitInst_rr(MachineInstOpcode: X86::PSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
20209 }
20210 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
20211 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
20212 }
20213 return Register();
20214}
20215
20216Register fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20217 if (RetVT.SimpleTy != MVT::v4i32)
20218 return Register();
20219 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20220 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20221 }
20222 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20223 return fastEmitInst_rr(MachineInstOpcode: X86::PSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
20224 }
20225 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20226 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
20227 }
20228 return Register();
20229}
20230
20231Register fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20232 if (RetVT.SimpleTy != MVT::v2i64)
20233 return Register();
20234 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20235 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20236 }
20237 return Register();
20238}
20239
20240Register fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20241 switch (VT.SimpleTy) {
20242 case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op1);
20243 case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op1);
20244 case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op1);
20245 default: return Register();
20246 }
20247}
20248
20249// FastEmit functions for X86ISD::VSRAV.
20250
20251Register fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20252 if (RetVT.SimpleTy != MVT::v8i16)
20253 return Register();
20254 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20255 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20256 }
20257 return Register();
20258}
20259
20260Register fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20261 if (RetVT.SimpleTy != MVT::v16i16)
20262 return Register();
20263 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20264 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20265 }
20266 return Register();
20267}
20268
20269Register fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20270 if (RetVT.SimpleTy != MVT::v32i16)
20271 return Register();
20272 if ((Subtarget->hasBWI())) {
20273 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20274 }
20275 return Register();
20276}
20277
20278Register fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20279 if (RetVT.SimpleTy != MVT::v4i32)
20280 return Register();
20281 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20282 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20283 }
20284 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20285 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20286 }
20287 return Register();
20288}
20289
20290Register fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20291 if (RetVT.SimpleTy != MVT::v8i32)
20292 return Register();
20293 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20294 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20295 }
20296 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20297 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20298 }
20299 return Register();
20300}
20301
20302Register fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20303 if (RetVT.SimpleTy != MVT::v16i32)
20304 return Register();
20305 if ((Subtarget->hasAVX512())) {
20306 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20307 }
20308 return Register();
20309}
20310
20311Register fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20312 if (RetVT.SimpleTy != MVT::v2i64)
20313 return Register();
20314 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20315 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20316 }
20317 return Register();
20318}
20319
20320Register fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20321 if (RetVT.SimpleTy != MVT::v4i64)
20322 return Register();
20323 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20324 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20325 }
20326 return Register();
20327}
20328
20329Register fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20330 if (RetVT.SimpleTy != MVT::v8i64)
20331 return Register();
20332 if ((Subtarget->hasAVX512())) {
20333 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20334 }
20335 return Register();
20336}
20337
20338Register fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20339 switch (VT.SimpleTy) {
20340 case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op1);
20341 case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op1);
20342 case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op1);
20343 case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op1);
20344 case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op1);
20345 case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op1);
20346 case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op1);
20347 case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op1);
20348 case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op1);
20349 default: return Register();
20350 }
20351}
20352
20353// FastEmit functions for X86ISD::VSRL.
20354
20355Register fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20356 if (RetVT.SimpleTy != MVT::v8i16)
20357 return Register();
20358 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20359 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20360 }
20361 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20362 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20363 }
20364 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
20365 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20366 }
20367 return Register();
20368}
20369
20370Register fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20371 if (RetVT.SimpleTy != MVT::v4i32)
20372 return Register();
20373 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20374 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20375 }
20376 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20377 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20378 }
20379 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20380 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20381 }
20382 return Register();
20383}
20384
20385Register fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20386 if (RetVT.SimpleTy != MVT::v2i64)
20387 return Register();
20388 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20389 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20390 }
20391 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20392 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20393 }
20394 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20395 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20396 }
20397 return Register();
20398}
20399
20400Register fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20401 switch (VT.SimpleTy) {
20402 case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op1);
20403 case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op1);
20404 case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op1);
20405 default: return Register();
20406 }
20407}
20408
20409// FastEmit functions for X86ISD::VSRLV.
20410
20411Register fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20412 if (RetVT.SimpleTy != MVT::v8i16)
20413 return Register();
20414 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20415 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20416 }
20417 return Register();
20418}
20419
20420Register fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20421 if (RetVT.SimpleTy != MVT::v16i16)
20422 return Register();
20423 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20424 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20425 }
20426 return Register();
20427}
20428
20429Register fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20430 if (RetVT.SimpleTy != MVT::v32i16)
20431 return Register();
20432 if ((Subtarget->hasBWI())) {
20433 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20434 }
20435 return Register();
20436}
20437
20438Register fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20439 if (RetVT.SimpleTy != MVT::v4i32)
20440 return Register();
20441 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20442 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20443 }
20444 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20445 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20446 }
20447 return Register();
20448}
20449
20450Register fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20451 if (RetVT.SimpleTy != MVT::v8i32)
20452 return Register();
20453 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20454 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20455 }
20456 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20457 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20458 }
20459 return Register();
20460}
20461
20462Register fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20463 if (RetVT.SimpleTy != MVT::v16i32)
20464 return Register();
20465 if ((Subtarget->hasAVX512())) {
20466 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20467 }
20468 return Register();
20469}
20470
20471Register fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20472 if (RetVT.SimpleTy != MVT::v2i64)
20473 return Register();
20474 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20475 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20476 }
20477 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20478 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
20479 }
20480 return Register();
20481}
20482
20483Register fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20484 if (RetVT.SimpleTy != MVT::v4i64)
20485 return Register();
20486 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20487 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20488 }
20489 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20490 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
20491 }
20492 return Register();
20493}
20494
20495Register fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20496 if (RetVT.SimpleTy != MVT::v8i64)
20497 return Register();
20498 if ((Subtarget->hasAVX512())) {
20499 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20500 }
20501 return Register();
20502}
20503
20504Register fastEmit_X86ISD_VSRLV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20505 switch (VT.SimpleTy) {
20506 case MVT::v8i16: return fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(RetVT, Op0, Op1);
20507 case MVT::v16i16: return fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(RetVT, Op0, Op1);
20508 case MVT::v32i16: return fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(RetVT, Op0, Op1);
20509 case MVT::v4i32: return fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(RetVT, Op0, Op1);
20510 case MVT::v8i32: return fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(RetVT, Op0, Op1);
20511 case MVT::v16i32: return fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(RetVT, Op0, Op1);
20512 case MVT::v2i64: return fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(RetVT, Op0, Op1);
20513 case MVT::v4i64: return fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(RetVT, Op0, Op1);
20514 case MVT::v8i64: return fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(RetVT, Op0, Op1);
20515 default: return Register();
20516 }
20517}
20518
20519// Top-level FastEmit function.
20520
20521Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
20522 switch (Opcode) {
20523 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
20524 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
20525 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
20526 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
20527 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
20528 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
20529 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
20530 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
20531 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
20532 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
20533 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
20534 case ISD::PDEP: return fastEmit_ISD_PDEP_rr(VT, RetVT, Op0, Op1);
20535 case ISD::PEXT: return fastEmit_ISD_PEXT_rr(VT, RetVT, Op0, Op1);
20536 case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op1);
20537 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
20538 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
20539 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
20540 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
20541 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
20542 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
20543 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
20544 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
20545 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
20546 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
20547 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
20548 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
20549 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
20550 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
20551 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
20552 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
20553 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
20554 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
20555 case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op1);
20556 case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op1);
20557 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op1);
20558 case X86ISD::BT: return fastEmit_X86ISD_BT_rr(VT, RetVT, Op0, Op1);
20559 case X86ISD::BZHI: return fastEmit_X86ISD_BZHI_rr(VT, RetVT, Op0, Op1);
20560 case X86ISD::CMP: return fastEmit_X86ISD_CMP_rr(VT, RetVT, Op0, Op1);
20561 case X86ISD::COMI: return fastEmit_X86ISD_COMI_rr(VT, RetVT, Op0, Op1);
20562 case X86ISD::COMX: return fastEmit_X86ISD_COMX_rr(VT, RetVT, Op0, Op1);
20563 case X86ISD::FADDS: return fastEmit_X86ISD_FADDS_rr(VT, RetVT, Op0, Op1);
20564 case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op1);
20565 case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op1);
20566 case X86ISD::FCMP: return fastEmit_X86ISD_FCMP_rr(VT, RetVT, Op0, Op1);
20567 case X86ISD::FDIVS: return fastEmit_X86ISD_FDIVS_rr(VT, RetVT, Op0, Op1);
20568 case X86ISD::FGETEXPS: return fastEmit_X86ISD_FGETEXPS_rr(VT, RetVT, Op0, Op1);
20569 case X86ISD::FGETEXPS_SAE: return fastEmit_X86ISD_FGETEXPS_SAE_rr(VT, RetVT, Op0, Op1);
20570 case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op1);
20571 case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op1);
20572 case X86ISD::FMAX: return fastEmit_X86ISD_FMAX_rr(VT, RetVT, Op0, Op1);
20573 case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op1);
20574 case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op1);
20575 case X86ISD::FMAXS_SAE: return fastEmit_X86ISD_FMAXS_SAE_rr(VT, RetVT, Op0, Op1);
20576 case X86ISD::FMAX_SAE: return fastEmit_X86ISD_FMAX_SAE_rr(VT, RetVT, Op0, Op1);
20577 case X86ISD::FMIN: return fastEmit_X86ISD_FMIN_rr(VT, RetVT, Op0, Op1);
20578 case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op1);
20579 case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op1);
20580 case X86ISD::FMINS_SAE: return fastEmit_X86ISD_FMINS_SAE_rr(VT, RetVT, Op0, Op1);
20581 case X86ISD::FMIN_SAE: return fastEmit_X86ISD_FMIN_SAE_rr(VT, RetVT, Op0, Op1);
20582 case X86ISD::FMULS: return fastEmit_X86ISD_FMULS_rr(VT, RetVT, Op0, Op1);
20583 case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op1);
20584 case X86ISD::FP80_ADD: return fastEmit_X86ISD_FP80_ADD_rr(VT, RetVT, Op0, Op1);
20585 case X86ISD::FSQRTS: return fastEmit_X86ISD_FSQRTS_rr(VT, RetVT, Op0, Op1);
20586 case X86ISD::FSUBS: return fastEmit_X86ISD_FSUBS_rr(VT, RetVT, Op0, Op1);
20587 case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op1);
20588 case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op1);
20589 case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op1);
20590 case X86ISD::HADDS: return fastEmit_X86ISD_HADDS_rr(VT, RetVT, Op0, Op1);
20591 case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op1);
20592 case X86ISD::HSUBS: return fastEmit_X86ISD_HSUBS_rr(VT, RetVT, Op0, Op1);
20593 case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op1);
20594 case X86ISD::KORTEST: return fastEmit_X86ISD_KORTEST_rr(VT, RetVT, Op0, Op1);
20595 case X86ISD::KTEST: return fastEmit_X86ISD_KTEST_rr(VT, RetVT, Op0, Op1);
20596 case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op1);
20597 case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op1);
20598 case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op1);
20599 case X86ISD::MOVSH: return fastEmit_X86ISD_MOVSH_rr(VT, RetVT, Op0, Op1);
20600 case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op1);
20601 case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op1);
20602 case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op1);
20603 case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op1);
20604 case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op1);
20605 case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op1);
20606 case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op1);
20607 case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op1);
20608 case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op1);
20609 case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op1);
20610 case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op1);
20611 case X86ISD::PTEST: return fastEmit_X86ISD_PTEST_rr(VT, RetVT, Op0, Op1);
20612 case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op1);
20613 case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op1);
20614 case X86ISD::SCALEF: return fastEmit_X86ISD_SCALEF_rr(VT, RetVT, Op0, Op1);
20615 case X86ISD::SCALEFS: return fastEmit_X86ISD_SCALEFS_rr(VT, RetVT, Op0, Op1);
20616 case X86ISD::STRICT_FCMP: return fastEmit_X86ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
20617 case X86ISD::STRICT_FCMPS: return fastEmit_X86ISD_STRICT_FCMPS_rr(VT, RetVT, Op0, Op1);
20618 case X86ISD::STRICT_FMAX: return fastEmit_X86ISD_STRICT_FMAX_rr(VT, RetVT, Op0, Op1);
20619 case X86ISD::STRICT_FMIN: return fastEmit_X86ISD_STRICT_FMIN_rr(VT, RetVT, Op0, Op1);
20620 case X86ISD::STRICT_FP80_ADD: return fastEmit_X86ISD_STRICT_FP80_ADD_rr(VT, RetVT, Op0, Op1);
20621 case X86ISD::TESTP: return fastEmit_X86ISD_TESTP_rr(VT, RetVT, Op0, Op1);
20622 case X86ISD::UCOMI: return fastEmit_X86ISD_UCOMI_rr(VT, RetVT, Op0, Op1);
20623 case X86ISD::UCOMX: return fastEmit_X86ISD_UCOMX_rr(VT, RetVT, Op0, Op1);
20624 case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op1);
20625 case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op1);
20626 case X86ISD::VCVT2PH2BF8: return fastEmit_X86ISD_VCVT2PH2BF8_rr(VT, RetVT, Op0, Op1);
20627 case X86ISD::VCVT2PH2BF8S: return fastEmit_X86ISD_VCVT2PH2BF8S_rr(VT, RetVT, Op0, Op1);
20628 case X86ISD::VCVT2PH2HF8: return fastEmit_X86ISD_VCVT2PH2HF8_rr(VT, RetVT, Op0, Op1);
20629 case X86ISD::VCVT2PH2HF8S: return fastEmit_X86ISD_VCVT2PH2HF8S_rr(VT, RetVT, Op0, Op1);
20630 case X86ISD::VFCMULC: return fastEmit_X86ISD_VFCMULC_rr(VT, RetVT, Op0, Op1);
20631 case X86ISD::VFCMULCSH: return fastEmit_X86ISD_VFCMULCSH_rr(VT, RetVT, Op0, Op1);
20632 case X86ISD::VFMULC: return fastEmit_X86ISD_VFMULC_rr(VT, RetVT, Op0, Op1);
20633 case X86ISD::VFMULCSH: return fastEmit_X86ISD_VFMULCSH_rr(VT, RetVT, Op0, Op1);
20634 case X86ISD::VFPROUND2: return fastEmit_X86ISD_VFPROUND2_rr(VT, RetVT, Op0, Op1);
20635 case X86ISD::VP2INTERSECT: return fastEmit_X86ISD_VP2INTERSECT_rr(VT, RetVT, Op0, Op1);
20636 case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op1);
20637 case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op1);
20638 case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op1);
20639 case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op1);
20640 case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op1);
20641 case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op1);
20642 case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op1);
20643 case X86ISD::VSHLV: return fastEmit_X86ISD_VSHLV_rr(VT, RetVT, Op0, Op1);
20644 case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op1);
20645 case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op1);
20646 case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op1);
20647 case X86ISD::VSRLV: return fastEmit_X86ISD_VSRLV_rr(VT, RetVT, Op0, Op1);
20648 default: return Register();
20649 }
20650}
20651
20652// FastEmit functions for ISD::ADD.
20653
20654Register fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20655 if (RetVT.SimpleTy != MVT::i8)
20656 return Register();
20657 if ((Subtarget->hasNDD())) {
20658 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20659 }
20660 if ((!Subtarget->hasNDD())) {
20661 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20662 }
20663 return Register();
20664}
20665
20666Register fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20667 if (RetVT.SimpleTy != MVT::i16)
20668 return Register();
20669 if ((Subtarget->hasNDD())) {
20670 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20671 }
20672 if ((!Subtarget->hasNDD())) {
20673 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20674 }
20675 return Register();
20676}
20677
20678Register fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20679 if (RetVT.SimpleTy != MVT::i32)
20680 return Register();
20681 if ((Subtarget->hasNDD())) {
20682 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20683 }
20684 if ((!Subtarget->hasNDD())) {
20685 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20686 }
20687 return Register();
20688}
20689
20690Register fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20691 switch (VT.SimpleTy) {
20692 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, imm1);
20693 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, imm1);
20694 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, imm1);
20695 default: return Register();
20696 }
20697}
20698
20699// FastEmit functions for ISD::AND.
20700
20701Register fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20702 if (RetVT.SimpleTy != MVT::i8)
20703 return Register();
20704 if ((Subtarget->hasNDD())) {
20705 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20706 }
20707 if ((!Subtarget->hasNDD())) {
20708 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20709 }
20710 return Register();
20711}
20712
20713Register fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20714 if (RetVT.SimpleTy != MVT::i16)
20715 return Register();
20716 if ((Subtarget->hasNDD())) {
20717 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20718 }
20719 if ((!Subtarget->hasNDD())) {
20720 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20721 }
20722 return Register();
20723}
20724
20725Register fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20726 if (RetVT.SimpleTy != MVT::i32)
20727 return Register();
20728 if ((Subtarget->hasNDD())) {
20729 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20730 }
20731 if ((!Subtarget->hasNDD())) {
20732 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20733 }
20734 return Register();
20735}
20736
20737Register fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20738 switch (VT.SimpleTy) {
20739 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, imm1);
20740 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, imm1);
20741 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, imm1);
20742 default: return Register();
20743 }
20744}
20745
20746// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
20747
20748Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20749 if (RetVT.SimpleTy != MVT::i32)
20750 return Register();
20751 if ((Subtarget->hasDQI())) {
20752 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDZrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20753 }
20754 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
20755 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRDrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20756 }
20757 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
20758 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20759 }
20760 return Register();
20761}
20762
20763Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20764 if (RetVT.SimpleTy != MVT::i64)
20765 return Register();
20766 if ((Subtarget->hasDQI())) {
20767 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQZrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20768 }
20769 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
20770 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRQrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20771 }
20772 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
20773 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20774 }
20775 return Register();
20776}
20777
20778Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20779 switch (VT.SimpleTy) {
20780 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, imm1);
20781 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, imm1);
20782 default: return Register();
20783 }
20784}
20785
20786// FastEmit functions for ISD::MUL.
20787
20788Register fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20789 if (RetVT.SimpleTy != MVT::i16)
20790 return Register();
20791 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL16rri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20792}
20793
20794Register fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20795 if (RetVT.SimpleTy != MVT::i32)
20796 return Register();
20797 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL32rri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20798}
20799
20800Register fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20801 switch (VT.SimpleTy) {
20802 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, imm1);
20803 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, imm1);
20804 default: return Register();
20805 }
20806}
20807
20808// FastEmit functions for ISD::OR.
20809
20810Register fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20811 if (RetVT.SimpleTy != MVT::i8)
20812 return Register();
20813 if ((Subtarget->hasNDD())) {
20814 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20815 }
20816 if ((!Subtarget->hasNDD())) {
20817 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20818 }
20819 return Register();
20820}
20821
20822Register fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20823 if (RetVT.SimpleTy != MVT::i16)
20824 return Register();
20825 if ((Subtarget->hasNDD())) {
20826 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20827 }
20828 if ((!Subtarget->hasNDD())) {
20829 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20830 }
20831 return Register();
20832}
20833
20834Register fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20835 if (RetVT.SimpleTy != MVT::i32)
20836 return Register();
20837 if ((Subtarget->hasNDD())) {
20838 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20839 }
20840 if ((!Subtarget->hasNDD())) {
20841 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20842 }
20843 return Register();
20844}
20845
20846Register fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20847 switch (VT.SimpleTy) {
20848 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, imm1);
20849 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, imm1);
20850 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, imm1);
20851 default: return Register();
20852 }
20853}
20854
20855// FastEmit functions for ISD::ROTL.
20856
20857Register fastEmit_ISD_ROTL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20858 if (RetVT.SimpleTy != MVT::i8)
20859 return Register();
20860 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20861 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20862 }
20863 if ((!Subtarget->hasNDD())) {
20864 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20865 }
20866 return Register();
20867}
20868
20869Register fastEmit_ISD_ROTL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20870 if (RetVT.SimpleTy != MVT::i16)
20871 return Register();
20872 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20873 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20874 }
20875 if ((!Subtarget->hasNDD())) {
20876 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20877 }
20878 return Register();
20879}
20880
20881Register fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20882 if (RetVT.SimpleTy != MVT::i32)
20883 return Register();
20884 if ((Subtarget->hasFastSHLDRotate())) {
20885 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20886 }
20887 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20888 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20889 }
20890 if ((!Subtarget->hasNDD())) {
20891 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20892 }
20893 return Register();
20894}
20895
20896Register fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20897 if (RetVT.SimpleTy != MVT::i64)
20898 return Register();
20899 if ((Subtarget->hasFastSHLDRotate())) {
20900 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20901 }
20902 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20903 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20904 }
20905 if ((!Subtarget->hasNDD())) {
20906 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20907 }
20908 return Register();
20909}
20910
20911Register fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20912 switch (VT.SimpleTy) {
20913 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_ri(RetVT, Op0, imm1);
20914 case MVT::i16: return fastEmit_ISD_ROTL_MVT_i16_ri(RetVT, Op0, imm1);
20915 case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, imm1);
20916 case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, imm1);
20917 default: return Register();
20918 }
20919}
20920
20921// FastEmit functions for ISD::ROTR.
20922
20923Register fastEmit_ISD_ROTR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20924 if (RetVT.SimpleTy != MVT::i8)
20925 return Register();
20926 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20927 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20928 }
20929 if ((!Subtarget->hasNDD())) {
20930 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20931 }
20932 return Register();
20933}
20934
20935Register fastEmit_ISD_ROTR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20936 if (RetVT.SimpleTy != MVT::i16)
20937 return Register();
20938 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20939 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20940 }
20941 if ((!Subtarget->hasNDD())) {
20942 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20943 }
20944 return Register();
20945}
20946
20947Register fastEmit_ISD_ROTR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20948 if (RetVT.SimpleTy != MVT::i32)
20949 return Register();
20950 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
20951 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri_EVEX, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20952 }
20953 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
20954 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20955 }
20956 if ((Subtarget->hasFastSHLDRotate())) {
20957 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20958 }
20959 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20960 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20961 }
20962 if ((!Subtarget->hasNDD())) {
20963 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20964 }
20965 return Register();
20966}
20967
20968Register fastEmit_ISD_ROTR_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20969 if (RetVT.SimpleTy != MVT::i64)
20970 return Register();
20971 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
20972 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri_EVEX, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20973 }
20974 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
20975 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20976 }
20977 if ((Subtarget->hasFastSHLDRotate())) {
20978 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20979 }
20980 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20981 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20982 }
20983 if ((!Subtarget->hasNDD())) {
20984 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20985 }
20986 return Register();
20987}
20988
20989Register fastEmit_ISD_ROTR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20990 switch (VT.SimpleTy) {
20991 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_ri(RetVT, Op0, imm1);
20992 case MVT::i16: return fastEmit_ISD_ROTR_MVT_i16_ri(RetVT, Op0, imm1);
20993 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri(RetVT, Op0, imm1);
20994 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_ri(RetVT, Op0, imm1);
20995 default: return Register();
20996 }
20997}
20998
20999// FastEmit functions for ISD::SHL.
21000
21001Register fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21002 if (RetVT.SimpleTy != MVT::i8)
21003 return Register();
21004 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21005 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21006 }
21007 if ((!Subtarget->hasNDD())) {
21008 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21009 }
21010 return Register();
21011}
21012
21013Register fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21014 if (RetVT.SimpleTy != MVT::i16)
21015 return Register();
21016 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21017 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21018 }
21019 if ((!Subtarget->hasNDD())) {
21020 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21021 }
21022 return Register();
21023}
21024
21025Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21026 if (RetVT.SimpleTy != MVT::i32)
21027 return Register();
21028 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21029 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21030 }
21031 if ((!Subtarget->hasNDD())) {
21032 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21033 }
21034 return Register();
21035}
21036
21037Register fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21038 if (RetVT.SimpleTy != MVT::i64)
21039 return Register();
21040 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21041 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21042 }
21043 if ((!Subtarget->hasNDD())) {
21044 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21045 }
21046 return Register();
21047}
21048
21049Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21050 switch (VT.SimpleTy) {
21051 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, imm1);
21052 case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, imm1);
21053 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
21054 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, imm1);
21055 default: return Register();
21056 }
21057}
21058
21059// FastEmit functions for ISD::SRA.
21060
21061Register fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21062 if (RetVT.SimpleTy != MVT::i8)
21063 return Register();
21064 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21065 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21066 }
21067 if ((!Subtarget->hasNDD())) {
21068 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21069 }
21070 return Register();
21071}
21072
21073Register fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21074 if (RetVT.SimpleTy != MVT::i16)
21075 return Register();
21076 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21077 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21078 }
21079 if ((!Subtarget->hasNDD())) {
21080 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21081 }
21082 return Register();
21083}
21084
21085Register fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21086 if (RetVT.SimpleTy != MVT::i32)
21087 return Register();
21088 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21089 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21090 }
21091 if ((!Subtarget->hasNDD())) {
21092 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21093 }
21094 return Register();
21095}
21096
21097Register fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21098 if (RetVT.SimpleTy != MVT::i64)
21099 return Register();
21100 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21101 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21102 }
21103 if ((!Subtarget->hasNDD())) {
21104 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21105 }
21106 return Register();
21107}
21108
21109Register fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21110 switch (VT.SimpleTy) {
21111 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, imm1);
21112 case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, imm1);
21113 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, imm1);
21114 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, imm1);
21115 default: return Register();
21116 }
21117}
21118
21119// FastEmit functions for ISD::SRL.
21120
21121Register fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21122 if (RetVT.SimpleTy != MVT::i8)
21123 return Register();
21124 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21125 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21126 }
21127 if ((!Subtarget->hasNDD())) {
21128 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21129 }
21130 return Register();
21131}
21132
21133Register fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21134 if (RetVT.SimpleTy != MVT::i16)
21135 return Register();
21136 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21137 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21138 }
21139 if ((!Subtarget->hasNDD())) {
21140 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21141 }
21142 return Register();
21143}
21144
21145Register fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21146 if (RetVT.SimpleTy != MVT::i32)
21147 return Register();
21148 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21149 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21150 }
21151 if ((!Subtarget->hasNDD())) {
21152 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21153 }
21154 return Register();
21155}
21156
21157Register fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21158 if (RetVT.SimpleTy != MVT::i64)
21159 return Register();
21160 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21161 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21162 }
21163 if ((!Subtarget->hasNDD())) {
21164 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21165 }
21166 return Register();
21167}
21168
21169Register fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21170 switch (VT.SimpleTy) {
21171 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, imm1);
21172 case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, imm1);
21173 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, imm1);
21174 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, imm1);
21175 default: return Register();
21176 }
21177}
21178
21179// FastEmit functions for ISD::SUB.
21180
21181Register fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21182 if (RetVT.SimpleTy != MVT::i8)
21183 return Register();
21184 if ((Subtarget->hasNDD())) {
21185 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21186 }
21187 if ((!Subtarget->hasNDD())) {
21188 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21189 }
21190 return Register();
21191}
21192
21193Register fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21194 if (RetVT.SimpleTy != MVT::i16)
21195 return Register();
21196 if ((Subtarget->hasNDD())) {
21197 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21198 }
21199 if ((!Subtarget->hasNDD())) {
21200 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21201 }
21202 return Register();
21203}
21204
21205Register fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21206 if (RetVT.SimpleTy != MVT::i32)
21207 return Register();
21208 if ((Subtarget->hasNDD())) {
21209 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21210 }
21211 if ((!Subtarget->hasNDD())) {
21212 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21213 }
21214 return Register();
21215}
21216
21217Register fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21218 switch (VT.SimpleTy) {
21219 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, imm1);
21220 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, imm1);
21221 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, imm1);
21222 default: return Register();
21223 }
21224}
21225
21226// FastEmit functions for ISD::XOR.
21227
21228Register fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21229 if (RetVT.SimpleTy != MVT::i8)
21230 return Register();
21231 if ((Subtarget->hasNDD())) {
21232 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21233 }
21234 if ((!Subtarget->hasNDD())) {
21235 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21236 }
21237 return Register();
21238}
21239
21240Register fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21241 if (RetVT.SimpleTy != MVT::i16)
21242 return Register();
21243 if ((Subtarget->hasNDD())) {
21244 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21245 }
21246 if ((!Subtarget->hasNDD())) {
21247 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21248 }
21249 return Register();
21250}
21251
21252Register fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21253 if (RetVT.SimpleTy != MVT::i32)
21254 return Register();
21255 if ((Subtarget->hasNDD())) {
21256 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21257 }
21258 if ((!Subtarget->hasNDD())) {
21259 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21260 }
21261 return Register();
21262}
21263
21264Register fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21265 switch (VT.SimpleTy) {
21266 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, imm1);
21267 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, imm1);
21268 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, imm1);
21269 default: return Register();
21270 }
21271}
21272
21273// FastEmit functions for X86ISD::BT.
21274
21275Register fastEmit_X86ISD_BT_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21276 if (RetVT.SimpleTy != MVT::i32)
21277 return Register();
21278 return fastEmitInst_ri(MachineInstOpcode: X86::BT16ri8, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21279}
21280
21281Register fastEmit_X86ISD_BT_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21282 if (RetVT.SimpleTy != MVT::i32)
21283 return Register();
21284 return fastEmitInst_ri(MachineInstOpcode: X86::BT32ri8, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21285}
21286
21287Register fastEmit_X86ISD_BT_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21288 if (RetVT.SimpleTy != MVT::i32)
21289 return Register();
21290 return fastEmitInst_ri(MachineInstOpcode: X86::BT64ri8, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21291}
21292
21293Register fastEmit_X86ISD_BT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21294 switch (VT.SimpleTy) {
21295 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_ri(RetVT, Op0, imm1);
21296 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_ri(RetVT, Op0, imm1);
21297 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_ri(RetVT, Op0, imm1);
21298 default: return Register();
21299 }
21300}
21301
21302// Top-level FastEmit function.
21303
21304Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
21305 if (VT == MVT::i64 && Predicate_i64immSExt32(Imm: imm1))
21306 if (Register Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, imm1))
21307 return Reg;
21308
21309 switch (Opcode) {
21310 case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, imm1);
21311 case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, imm1);
21312 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
21313 case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, imm1);
21314 case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, imm1);
21315 case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, imm1);
21316 case ISD::ROTR: return fastEmit_ISD_ROTR_ri(VT, RetVT, Op0, imm1);
21317 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
21318 case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, imm1);
21319 case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, imm1);
21320 case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, imm1);
21321 case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, imm1);
21322 case X86ISD::BT: return fastEmit_X86ISD_BT_ri(VT, RetVT, Op0, imm1);
21323 default: return Register();
21324 }
21325}
21326
21327// FastEmit functions for ISD::ADD.
21328
21329Register fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21330 if (RetVT.SimpleTy != MVT::i64)
21331 return Register();
21332 if ((Subtarget->hasNDD())) {
21333 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21334 }
21335 if ((!Subtarget->hasNDD())) {
21336 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21337 }
21338 return Register();
21339}
21340
21341Register fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21342 switch (VT.SimpleTy) {
21343 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21344 default: return Register();
21345 }
21346}
21347
21348// FastEmit functions for ISD::AND.
21349
21350Register fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21351 if (RetVT.SimpleTy != MVT::i64)
21352 return Register();
21353 if ((Subtarget->hasNDD())) {
21354 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21355 }
21356 if ((!Subtarget->hasNDD())) {
21357 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21358 }
21359 return Register();
21360}
21361
21362Register fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21363 switch (VT.SimpleTy) {
21364 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21365 default: return Register();
21366 }
21367}
21368
21369// FastEmit functions for ISD::MUL.
21370
21371Register fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21372 if (RetVT.SimpleTy != MVT::i64)
21373 return Register();
21374 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL64rri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21375}
21376
21377Register fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21378 switch (VT.SimpleTy) {
21379 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21380 default: return Register();
21381 }
21382}
21383
21384// FastEmit functions for ISD::OR.
21385
21386Register fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21387 if (RetVT.SimpleTy != MVT::i64)
21388 return Register();
21389 if ((Subtarget->hasNDD())) {
21390 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21391 }
21392 if ((!Subtarget->hasNDD())) {
21393 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21394 }
21395 return Register();
21396}
21397
21398Register fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21399 switch (VT.SimpleTy) {
21400 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21401 default: return Register();
21402 }
21403}
21404
21405// FastEmit functions for ISD::SUB.
21406
21407Register fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21408 if (RetVT.SimpleTy != MVT::i64)
21409 return Register();
21410 if ((Subtarget->hasNDD())) {
21411 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21412 }
21413 if ((!Subtarget->hasNDD())) {
21414 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21415 }
21416 return Register();
21417}
21418
21419Register fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21420 switch (VT.SimpleTy) {
21421 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21422 default: return Register();
21423 }
21424}
21425
21426// FastEmit functions for ISD::XOR.
21427
21428Register fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21429 if (RetVT.SimpleTy != MVT::i64)
21430 return Register();
21431 if ((Subtarget->hasNDD())) {
21432 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21433 }
21434 if ((!Subtarget->hasNDD())) {
21435 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21436 }
21437 return Register();
21438}
21439
21440Register fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21441 switch (VT.SimpleTy) {
21442 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21443 default: return Register();
21444 }
21445}
21446
21447// Top-level FastEmit function.
21448
21449Register fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
21450 switch (Opcode) {
21451 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21452 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21453 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21454 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21455 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21456 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21457 default: return Register();
21458 }
21459}
21460
21461// FastEmit functions for ISD::Constant.
21462
21463Register fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) {
21464 if (RetVT.SimpleTy != MVT::i8)
21465 return Register();
21466 return fastEmitInst_i(MachineInstOpcode: X86::MOV8ri, RC: &X86::GR8RegClass, Imm: imm0);
21467}
21468
21469Register fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) {
21470 if (RetVT.SimpleTy != MVT::i16)
21471 return Register();
21472 return fastEmitInst_i(MachineInstOpcode: X86::MOV16ri, RC: &X86::GR16RegClass, Imm: imm0);
21473}
21474
21475Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
21476 if (RetVT.SimpleTy != MVT::i32)
21477 return Register();
21478 return fastEmitInst_i(MachineInstOpcode: X86::MOV32ri, RC: &X86::GR32RegClass, Imm: imm0);
21479}
21480
21481Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
21482 if (RetVT.SimpleTy != MVT::i64)
21483 return Register();
21484 return fastEmitInst_i(MachineInstOpcode: X86::MOV64ri, RC: &X86::GR64RegClass, Imm: imm0);
21485}
21486
21487Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
21488 switch (VT.SimpleTy) {
21489 case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0);
21490 case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0);
21491 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
21492 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
21493 default: return Register();
21494 }
21495}
21496
21497// Top-level FastEmit function.
21498
21499Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
21500 switch (Opcode) {
21501 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
21502 default: return Register();
21503 }
21504}
21505
21506