1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the X86 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_AndMask64(int64_t Imm) {
12
13 return isMask_64(Value: Imm) && !isUInt<32>(x: Imm);
14
15}
16static bool Predicate_BTRMask64(int64_t Imm) {
17
18 return !isUInt<32>(x: Imm) && !isInt<32>(x: Imm) && isPowerOf2_64(Value: ~Imm);
19
20}
21static bool Predicate_BTCBTSMask64(int64_t Imm) {
22
23 return !isInt<32>(x: Imm) && isPowerOf2_64(Value: Imm);
24
25}
26static bool Predicate_i64immSExt32(int64_t Imm) {
27 return isInt<32>(x: Imm);
28}
29
30
31// FastEmit functions for ISD::ABS.
32
33Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
34 if (RetVT.SimpleTy != MVT::v16i8)
35 return Register();
36 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
37 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ128rr, RC: &X86::VR128XRegClass, Op0);
38 }
39 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
40 return fastEmitInst_r(MachineInstOpcode: X86::PABSBrr, RC: &X86::VR128RegClass, Op0);
41 }
42 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
43 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBrr, RC: &X86::VR128RegClass, Op0);
44 }
45 return Register();
46}
47
48Register fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, Register Op0) {
49 if (RetVT.SimpleTy != MVT::v32i8)
50 return Register();
51 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
52 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ256rr, RC: &X86::VR256XRegClass, Op0);
53 }
54 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
55 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBYrr, RC: &X86::VR256RegClass, Op0);
56 }
57 return Register();
58}
59
60Register fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, Register Op0) {
61 if (RetVT.SimpleTy != MVT::v64i8)
62 return Register();
63 if ((Subtarget->hasBWI())) {
64 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZrr, RC: &X86::VR512RegClass, Op0);
65 }
66 return Register();
67}
68
69Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
70 if (RetVT.SimpleTy != MVT::v8i16)
71 return Register();
72 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
73 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ128rr, RC: &X86::VR128XRegClass, Op0);
74 }
75 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
76 return fastEmitInst_r(MachineInstOpcode: X86::PABSWrr, RC: &X86::VR128RegClass, Op0);
77 }
78 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
79 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWrr, RC: &X86::VR128RegClass, Op0);
80 }
81 return Register();
82}
83
84Register fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, Register Op0) {
85 if (RetVT.SimpleTy != MVT::v16i16)
86 return Register();
87 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
88 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ256rr, RC: &X86::VR256XRegClass, Op0);
89 }
90 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
91 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWYrr, RC: &X86::VR256RegClass, Op0);
92 }
93 return Register();
94}
95
96Register fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, Register Op0) {
97 if (RetVT.SimpleTy != MVT::v32i16)
98 return Register();
99 if ((Subtarget->hasBWI())) {
100 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZrr, RC: &X86::VR512RegClass, Op0);
101 }
102 return Register();
103}
104
105Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
106 if (RetVT.SimpleTy != MVT::v4i32)
107 return Register();
108 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
109 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ128rr, RC: &X86::VR128XRegClass, Op0);
110 }
111 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
112 return fastEmitInst_r(MachineInstOpcode: X86::PABSDrr, RC: &X86::VR128RegClass, Op0);
113 }
114 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
115 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDrr, RC: &X86::VR128RegClass, Op0);
116 }
117 return Register();
118}
119
120Register fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, Register Op0) {
121 if (RetVT.SimpleTy != MVT::v8i32)
122 return Register();
123 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
124 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ256rr, RC: &X86::VR256XRegClass, Op0);
125 }
126 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
127 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDYrr, RC: &X86::VR256RegClass, Op0);
128 }
129 return Register();
130}
131
132Register fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, Register Op0) {
133 if (RetVT.SimpleTy != MVT::v16i32)
134 return Register();
135 if ((Subtarget->hasAVX512())) {
136 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZrr, RC: &X86::VR512RegClass, Op0);
137 }
138 return Register();
139}
140
141Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
142 if (RetVT.SimpleTy != MVT::v2i64)
143 return Register();
144 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
145 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ128rr, RC: &X86::VR128XRegClass, Op0);
146 }
147 return Register();
148}
149
150Register fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, Register Op0) {
151 if (RetVT.SimpleTy != MVT::v4i64)
152 return Register();
153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
154 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ256rr, RC: &X86::VR256XRegClass, Op0);
155 }
156 return Register();
157}
158
159Register fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, Register Op0) {
160 if (RetVT.SimpleTy != MVT::v8i64)
161 return Register();
162 if ((Subtarget->hasAVX512())) {
163 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZrr, RC: &X86::VR512RegClass, Op0);
164 }
165 return Register();
166}
167
168Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
169 switch (VT.SimpleTy) {
170 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
171 case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0);
172 case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0);
173 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
174 case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0);
175 case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0);
176 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
177 case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0);
178 case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0);
179 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
180 case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0);
181 case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0);
182 default: return Register();
183 }
184}
185
186// FastEmit functions for ISD::ANY_EXTEND.
187
188Register fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
189 if (RetVT.SimpleTy != MVT::i32)
190 return Register();
191 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
192}
193
194Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
195 switch (VT.SimpleTy) {
196 case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0);
197 default: return Register();
198 }
199}
200
201// FastEmit functions for ISD::BITCAST.
202
203Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
204 if (RetVT.SimpleTy != MVT::f32)
205 return Register();
206 if ((Subtarget->hasAVX512())) {
207 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSZrr, RC: &X86::FR32XRegClass, Op0);
208 }
209 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
210 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
211 }
212 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
213 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
214 }
215 return Register();
216}
217
218Register fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Register Op0) {
219 if ((Subtarget->hasAVX512())) {
220 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDZrr, RC: &X86::FR64XRegClass, Op0);
221 }
222 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
223 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toSDrr, RC: &X86::FR64RegClass, Op0);
224 }
225 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
226 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDrr, RC: &X86::FR64RegClass, Op0);
227 }
228 return Register();
229}
230
231Register fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Register Op0) {
232 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
233 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64to64rr, RC: &X86::VR64RegClass, Op0);
234 }
235 return Register();
236}
237
238Register fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, Register Op0) {
239switch (RetVT.SimpleTy) {
240 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0);
241 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0);
242 default: return Register();
243}
244}
245
246Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
247 if (RetVT.SimpleTy != MVT::i32)
248 return Register();
249 if ((Subtarget->hasAVX512())) {
250 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIZrr, RC: &X86::GR32RegClass, Op0);
251 }
252 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
253 return fastEmitInst_r(MachineInstOpcode: X86::MOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
254 }
255 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
256 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
257 }
258 return Register();
259}
260
261Register fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Register Op0) {
262 if ((Subtarget->hasAVX512())) {
263 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64Zrr, RC: &X86::GR64RegClass, Op0);
264 }
265 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
266 return fastEmitInst_r(MachineInstOpcode: X86::MOVSDto64rr, RC: &X86::GR64RegClass, Op0);
267 }
268 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
269 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64rr, RC: &X86::GR64RegClass, Op0);
270 }
271 return Register();
272}
273
274Register fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Register Op0) {
275 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
276 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVFR642Qrr, RC: &X86::VR64RegClass, Op0);
277 }
278 return Register();
279}
280
281Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
282switch (RetVT.SimpleTy) {
283 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0);
284 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0);
285 default: return Register();
286}
287}
288
289Register fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Register Op0) {
290 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
291 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64from64rr, RC: &X86::GR64RegClass, Op0);
292 }
293 return Register();
294}
295
296Register fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Register Op0) {
297 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
298 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2FR64rr, RC: &X86::FR64RegClass, Op0);
299 }
300 return Register();
301}
302
303Register fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, Register Op0) {
304switch (RetVT.SimpleTy) {
305 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0);
306 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0);
307 default: return Register();
308}
309}
310
311Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
312 switch (VT.SimpleTy) {
313 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
314 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0);
315 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
316 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
317 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0);
318 default: return Register();
319 }
320}
321
322// FastEmit functions for ISD::BRIND.
323
324Register fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, Register Op0) {
325 if (RetVT.SimpleTy != MVT::isVoid)
326 return Register();
327 if ((!Subtarget->is64Bit())) {
328 return fastEmitInst_r(MachineInstOpcode: X86::JMP16r, RC: &X86::GR16RegClass, Op0);
329 }
330 return Register();
331}
332
333Register fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
334 if (RetVT.SimpleTy != MVT::isVoid)
335 return Register();
336 if ((!Subtarget->is64Bit())) {
337 return fastEmitInst_r(MachineInstOpcode: X86::JMP32r, RC: &X86::GR32RegClass, Op0);
338 }
339 return Register();
340}
341
342Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
343 if (RetVT.SimpleTy != MVT::isVoid)
344 return Register();
345 if ((Subtarget->is64Bit())) {
346 return fastEmitInst_r(MachineInstOpcode: X86::JMP64r, RC: &X86::GR64RegClass, Op0);
347 }
348 return Register();
349}
350
351Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
352 switch (VT.SimpleTy) {
353 case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0);
354 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
355 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
356 default: return Register();
357 }
358}
359
360// FastEmit functions for ISD::BSWAP.
361
362Register fastEmit_ISD_BSWAP_MVT_i16_r(MVT RetVT, Register Op0) {
363 if (RetVT.SimpleTy != MVT::i16)
364 return Register();
365 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
366 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE16rr, RC: &X86::GR16RegClass, Op0);
367 }
368 return Register();
369}
370
371Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
372 if (RetVT.SimpleTy != MVT::i32)
373 return Register();
374 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
375 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE32rr, RC: &X86::GR32RegClass, Op0);
376 }
377 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
378 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP32r, RC: &X86::GR32RegClass, Op0);
379 }
380 return Register();
381}
382
383Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
384 if (RetVT.SimpleTy != MVT::i64)
385 return Register();
386 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
387 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE64rr, RC: &X86::GR64RegClass, Op0);
388 }
389 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
390 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP64r, RC: &X86::GR64RegClass, Op0);
391 }
392 return Register();
393}
394
395Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
396 switch (VT.SimpleTy) {
397 case MVT::i16: return fastEmit_ISD_BSWAP_MVT_i16_r(RetVT, Op0);
398 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
399 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
400 default: return Register();
401 }
402}
403
404// FastEmit functions for ISD::CTLZ.
405
406Register fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, Register Op0) {
407 if (RetVT.SimpleTy != MVT::i16)
408 return Register();
409 if ((Subtarget->hasLZCNT())) {
410 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT16rr, RC: &X86::GR16RegClass, Op0);
411 }
412 return Register();
413}
414
415Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
416 if (RetVT.SimpleTy != MVT::i32)
417 return Register();
418 if ((Subtarget->hasLZCNT())) {
419 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT32rr, RC: &X86::GR32RegClass, Op0);
420 }
421 return Register();
422}
423
424Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
425 if (RetVT.SimpleTy != MVT::i64)
426 return Register();
427 if ((Subtarget->hasLZCNT())) {
428 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT64rr, RC: &X86::GR64RegClass, Op0);
429 }
430 return Register();
431}
432
433Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
434 if (RetVT.SimpleTy != MVT::v4i32)
435 return Register();
436 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
437 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
438 }
439 return Register();
440}
441
442Register fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, Register Op0) {
443 if (RetVT.SimpleTy != MVT::v8i32)
444 return Register();
445 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
446 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
447 }
448 return Register();
449}
450
451Register fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, Register Op0) {
452 if (RetVT.SimpleTy != MVT::v16i32)
453 return Register();
454 if ((Subtarget->hasCDI())) {
455 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZrr, RC: &X86::VR512RegClass, Op0);
456 }
457 return Register();
458}
459
460Register fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, Register Op0) {
461 if (RetVT.SimpleTy != MVT::v2i64)
462 return Register();
463 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
464 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
465 }
466 return Register();
467}
468
469Register fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, Register Op0) {
470 if (RetVT.SimpleTy != MVT::v4i64)
471 return Register();
472 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
473 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
474 }
475 return Register();
476}
477
478Register fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, Register Op0) {
479 if (RetVT.SimpleTy != MVT::v8i64)
480 return Register();
481 if ((Subtarget->hasCDI())) {
482 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZrr, RC: &X86::VR512RegClass, Op0);
483 }
484 return Register();
485}
486
487Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
488 switch (VT.SimpleTy) {
489 case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0);
490 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
491 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
492 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
493 case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0);
494 case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0);
495 case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0);
496 case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0);
497 case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0);
498 default: return Register();
499 }
500}
501
502// FastEmit functions for ISD::CTPOP.
503
504Register fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, Register Op0) {
505 if (RetVT.SimpleTy != MVT::i16)
506 return Register();
507 if ((Subtarget->hasPOPCNT())) {
508 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT16rr, RC: &X86::GR16RegClass, Op0);
509 }
510 return Register();
511}
512
513Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
514 if (RetVT.SimpleTy != MVT::i32)
515 return Register();
516 if ((Subtarget->hasPOPCNT())) {
517 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT32rr, RC: &X86::GR32RegClass, Op0);
518 }
519 return Register();
520}
521
522Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
523 if (RetVT.SimpleTy != MVT::i64)
524 return Register();
525 if ((Subtarget->hasPOPCNT())) {
526 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT64rr, RC: &X86::GR64RegClass, Op0);
527 }
528 return Register();
529}
530
531Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
532 if (RetVT.SimpleTy != MVT::v16i8)
533 return Register();
534 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
535 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ128rr, RC: &X86::VR128XRegClass, Op0);
536 }
537 return Register();
538}
539
540Register fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, Register Op0) {
541 if (RetVT.SimpleTy != MVT::v32i8)
542 return Register();
543 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
544 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ256rr, RC: &X86::VR256XRegClass, Op0);
545 }
546 return Register();
547}
548
549Register fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, Register Op0) {
550 if (RetVT.SimpleTy != MVT::v64i8)
551 return Register();
552 if ((Subtarget->hasBITALG())) {
553 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZrr, RC: &X86::VR512RegClass, Op0);
554 }
555 return Register();
556}
557
558Register fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, Register Op0) {
559 if (RetVT.SimpleTy != MVT::v8i16)
560 return Register();
561 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
562 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ128rr, RC: &X86::VR128XRegClass, Op0);
563 }
564 return Register();
565}
566
567Register fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, Register Op0) {
568 if (RetVT.SimpleTy != MVT::v16i16)
569 return Register();
570 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
571 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ256rr, RC: &X86::VR256XRegClass, Op0);
572 }
573 return Register();
574}
575
576Register fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, Register Op0) {
577 if (RetVT.SimpleTy != MVT::v32i16)
578 return Register();
579 if ((Subtarget->hasBITALG())) {
580 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZrr, RC: &X86::VR512RegClass, Op0);
581 }
582 return Register();
583}
584
585Register fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, Register Op0) {
586 if (RetVT.SimpleTy != MVT::v4i32)
587 return Register();
588 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
589 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
590 }
591 return Register();
592}
593
594Register fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, Register Op0) {
595 if (RetVT.SimpleTy != MVT::v8i32)
596 return Register();
597 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
598 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
599 }
600 return Register();
601}
602
603Register fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, Register Op0) {
604 if (RetVT.SimpleTy != MVT::v16i32)
605 return Register();
606 if ((Subtarget->hasVPOPCNTDQ())) {
607 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZrr, RC: &X86::VR512RegClass, Op0);
608 }
609 return Register();
610}
611
612Register fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, Register Op0) {
613 if (RetVT.SimpleTy != MVT::v2i64)
614 return Register();
615 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
616 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
617 }
618 return Register();
619}
620
621Register fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, Register Op0) {
622 if (RetVT.SimpleTy != MVT::v4i64)
623 return Register();
624 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
625 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
626 }
627 return Register();
628}
629
630Register fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, Register Op0) {
631 if (RetVT.SimpleTy != MVT::v8i64)
632 return Register();
633 if ((Subtarget->hasVPOPCNTDQ())) {
634 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZrr, RC: &X86::VR512RegClass, Op0);
635 }
636 return Register();
637}
638
639Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
640 switch (VT.SimpleTy) {
641 case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0);
642 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
643 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
644 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
645 case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0);
646 case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0);
647 case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0);
648 case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0);
649 case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0);
650 case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0);
651 case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0);
652 case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0);
653 case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0);
654 case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0);
655 case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0);
656 default: return Register();
657 }
658}
659
660// FastEmit functions for ISD::CTTZ.
661
662Register fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, Register Op0) {
663 if (RetVT.SimpleTy != MVT::i16)
664 return Register();
665 if ((Subtarget->hasBMI())) {
666 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT16rr, RC: &X86::GR16RegClass, Op0);
667 }
668 return Register();
669}
670
671Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
672 if (RetVT.SimpleTy != MVT::i32)
673 return Register();
674 if ((Subtarget->hasBMI())) {
675 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT32rr, RC: &X86::GR32RegClass, Op0);
676 }
677 return Register();
678}
679
680Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
681 if (RetVT.SimpleTy != MVT::i64)
682 return Register();
683 if ((Subtarget->hasBMI())) {
684 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT64rr, RC: &X86::GR64RegClass, Op0);
685 }
686 return Register();
687}
688
689Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
690 switch (VT.SimpleTy) {
691 case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0);
692 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
693 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
694 default: return Register();
695 }
696}
697
698// FastEmit functions for ISD::FABS.
699
700Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
701 if (RetVT.SimpleTy != MVT::f32)
702 return Register();
703 if ((!Subtarget->hasSSE1())) {
704 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp32, RC: &X86::RFP32RegClass, Op0);
705 }
706 return Register();
707}
708
709Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
710 if (RetVT.SimpleTy != MVT::f64)
711 return Register();
712 if ((!Subtarget->hasSSE2())) {
713 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp64, RC: &X86::RFP64RegClass, Op0);
714 }
715 return Register();
716}
717
718Register fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, Register Op0) {
719 if (RetVT.SimpleTy != MVT::f80)
720 return Register();
721 if ((Subtarget->hasX87())) {
722 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp80, RC: &X86::RFP80RegClass, Op0);
723 }
724 return Register();
725}
726
727Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
728 switch (VT.SimpleTy) {
729 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
730 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
731 case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0);
732 default: return Register();
733 }
734}
735
736// FastEmit functions for ISD::FNEG.
737
738Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
739 if (RetVT.SimpleTy != MVT::f32)
740 return Register();
741 if ((!Subtarget->hasSSE1())) {
742 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp32, RC: &X86::RFP32RegClass, Op0);
743 }
744 return Register();
745}
746
747Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
748 if (RetVT.SimpleTy != MVT::f64)
749 return Register();
750 if ((!Subtarget->hasSSE2())) {
751 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp64, RC: &X86::RFP64RegClass, Op0);
752 }
753 return Register();
754}
755
756Register fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, Register Op0) {
757 if (RetVT.SimpleTy != MVT::f80)
758 return Register();
759 if ((Subtarget->hasX87())) {
760 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp80, RC: &X86::RFP80RegClass, Op0);
761 }
762 return Register();
763}
764
765Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
766 switch (VT.SimpleTy) {
767 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
768 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
769 case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0);
770 default: return Register();
771 }
772}
773
774// FastEmit functions for ISD::FP_EXTEND.
775
776Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
777 if (RetVT.SimpleTy != MVT::f64)
778 return Register();
779 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
780 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
781 }
782 return Register();
783}
784
785Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Register Op0) {
786 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
787 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
788 }
789 return Register();
790}
791
792Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Register Op0) {
793 if ((Subtarget->hasFP16())) {
794 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
795 }
796 return Register();
797}
798
799Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(MVT RetVT, Register Op0) {
800switch (RetVT.SimpleTy) {
801 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
802 case MVT::v8f64: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
803 default: return Register();
804}
805}
806
807Register fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(MVT RetVT, Register Op0) {
808 if (RetVT.SimpleTy != MVT::v16f32)
809 return Register();
810 if ((Subtarget->hasFP16())) {
811 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
812 }
813 return Register();
814}
815
816Register fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, Register Op0) {
817 if (RetVT.SimpleTy != MVT::v4f64)
818 return Register();
819 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
820 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
821 }
822 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
823 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
824 }
825 return Register();
826}
827
828Register fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, Register Op0) {
829 if (RetVT.SimpleTy != MVT::v8f64)
830 return Register();
831 if ((Subtarget->hasAVX512())) {
832 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
833 }
834 return Register();
835}
836
837Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
838 switch (VT.SimpleTy) {
839 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
840 case MVT::v8f16: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
841 case MVT::v16f16: return fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
842 case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
843 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
844 default: return Register();
845 }
846}
847
848// FastEmit functions for ISD::FP_ROUND.
849
850Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
851 if (RetVT.SimpleTy != MVT::f32)
852 return Register();
853 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
854 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
855 }
856 return Register();
857}
858
859Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
860 switch (VT.SimpleTy) {
861 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
862 default: return Register();
863 }
864}
865
866// FastEmit functions for ISD::FP_TO_SINT.
867
868Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
869 if ((Subtarget->hasFP16())) {
870 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
871 }
872 return Register();
873}
874
875Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
876 if ((Subtarget->hasFP16())) {
877 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
878 }
879 return Register();
880}
881
882Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
883switch (RetVT.SimpleTy) {
884 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
885 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
886 default: return Register();
887}
888}
889
890Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
891 if ((Subtarget->hasAVX512())) {
892 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
893 }
894 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
895 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
896 }
897 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
898 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
899 }
900 return Register();
901}
902
903Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
904 if ((Subtarget->hasAVX512())) {
905 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
906 }
907 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
908 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
909 }
910 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
911 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
912 }
913 return Register();
914}
915
916Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
917switch (RetVT.SimpleTy) {
918 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
919 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
920 default: return Register();
921}
922}
923
924Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
925 if ((Subtarget->hasAVX512())) {
926 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
927 }
928 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
929 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
930 }
931 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
932 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
933 }
934 return Register();
935}
936
937Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
938 if ((Subtarget->hasAVX512())) {
939 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
940 }
941 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
942 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
943 }
944 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
945 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
946 }
947 return Register();
948}
949
950Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
951switch (RetVT.SimpleTy) {
952 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
953 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
954 default: return Register();
955}
956}
957
958Register fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
959 if (RetVT.SimpleTy != MVT::v4i32)
960 return Register();
961 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
963 }
964 return Register();
965}
966
967Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
968 switch (VT.SimpleTy) {
969 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
970 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
971 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
972 case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
973 default: return Register();
974 }
975}
976
977// FastEmit functions for ISD::FP_TO_UINT.
978
979Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
980 if ((Subtarget->hasFP16())) {
981 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
982 }
983 return Register();
984}
985
986Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
987 if ((Subtarget->hasFP16())) {
988 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
989 }
990 return Register();
991}
992
993Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
994switch (RetVT.SimpleTy) {
995 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
996 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
997 default: return Register();
998}
999}
1000
1001Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
1002 if ((Subtarget->hasAVX512())) {
1003 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
1004 }
1005 return Register();
1006}
1007
1008Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
1009 if ((Subtarget->hasAVX512())) {
1010 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1011 }
1012 return Register();
1013}
1014
1015Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
1016switch (RetVT.SimpleTy) {
1017 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
1018 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
1019 default: return Register();
1020}
1021}
1022
1023Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
1024 if ((Subtarget->hasAVX512())) {
1025 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
1026 }
1027 return Register();
1028}
1029
1030Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
1031 if ((Subtarget->hasAVX512())) {
1032 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1033 }
1034 return Register();
1035}
1036
1037Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
1038switch (RetVT.SimpleTy) {
1039 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
1040 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
1041 default: return Register();
1042}
1043}
1044
1045Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
1046 switch (VT.SimpleTy) {
1047 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
1048 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
1049 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
1050 default: return Register();
1051 }
1052}
1053
1054// FastEmit functions for ISD::FSQRT.
1055
1056Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
1057 if (RetVT.SimpleTy != MVT::f32)
1058 return Register();
1059 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1060 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
1061 }
1062 if ((!Subtarget->hasSSE1())) {
1063 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
1064 }
1065 return Register();
1066}
1067
1068Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
1069 if (RetVT.SimpleTy != MVT::f64)
1070 return Register();
1071 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1072 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
1073 }
1074 if ((!Subtarget->hasSSE2())) {
1075 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
1076 }
1077 return Register();
1078}
1079
1080Register fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, Register Op0) {
1081 if (RetVT.SimpleTy != MVT::f80)
1082 return Register();
1083 if ((Subtarget->hasX87())) {
1084 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
1085 }
1086 return Register();
1087}
1088
1089Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1090 if (RetVT.SimpleTy != MVT::v8f16)
1091 return Register();
1092 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1093 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
1094 }
1095 return Register();
1096}
1097
1098Register fastEmit_ISD_FSQRT_MVT_v16f16_r(MVT RetVT, Register Op0) {
1099 if (RetVT.SimpleTy != MVT::v16f16)
1100 return Register();
1101 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1102 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
1103 }
1104 return Register();
1105}
1106
1107Register fastEmit_ISD_FSQRT_MVT_v32f16_r(MVT RetVT, Register Op0) {
1108 if (RetVT.SimpleTy != MVT::v32f16)
1109 return Register();
1110 if ((Subtarget->hasFP16())) {
1111 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
1112 }
1113 return Register();
1114}
1115
1116Register fastEmit_ISD_FSQRT_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1117 if (RetVT.SimpleTy != MVT::v8bf16)
1118 return Register();
1119 if ((Subtarget->hasAVX10_2())) {
1120 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
1121 }
1122 return Register();
1123}
1124
1125Register fastEmit_ISD_FSQRT_MVT_v16bf16_r(MVT RetVT, Register Op0) {
1126 if (RetVT.SimpleTy != MVT::v16bf16)
1127 return Register();
1128 if ((Subtarget->hasAVX10_2())) {
1129 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
1130 }
1131 return Register();
1132}
1133
1134Register fastEmit_ISD_FSQRT_MVT_v32bf16_r(MVT RetVT, Register Op0) {
1135 if (RetVT.SimpleTy != MVT::v32bf16)
1136 return Register();
1137 if ((Subtarget->hasAVX10_2())) {
1138 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
1139 }
1140 return Register();
1141}
1142
1143Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1144 if (RetVT.SimpleTy != MVT::v4f32)
1145 return Register();
1146 if ((Subtarget->hasVLX())) {
1147 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
1148 }
1149 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1150 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
1151 }
1152 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1153 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
1154 }
1155 return Register();
1156}
1157
1158Register fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1159 if (RetVT.SimpleTy != MVT::v8f32)
1160 return Register();
1161 if ((Subtarget->hasVLX())) {
1162 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
1163 }
1164 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1165 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
1166 }
1167 return Register();
1168}
1169
1170Register fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, Register Op0) {
1171 if (RetVT.SimpleTy != MVT::v16f32)
1172 return Register();
1173 if ((Subtarget->hasAVX512())) {
1174 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
1175 }
1176 return Register();
1177}
1178
1179Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1180 if (RetVT.SimpleTy != MVT::v2f64)
1181 return Register();
1182 if ((Subtarget->hasVLX())) {
1183 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
1184 }
1185 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1186 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
1187 }
1188 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1189 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
1190 }
1191 return Register();
1192}
1193
1194Register fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1195 if (RetVT.SimpleTy != MVT::v4f64)
1196 return Register();
1197 if ((Subtarget->hasVLX())) {
1198 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
1199 }
1200 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1201 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
1202 }
1203 return Register();
1204}
1205
1206Register fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1207 if (RetVT.SimpleTy != MVT::v8f64)
1208 return Register();
1209 if ((Subtarget->hasAVX512())) {
1210 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
1211 }
1212 return Register();
1213}
1214
1215Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
1216 switch (VT.SimpleTy) {
1217 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
1218 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
1219 case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0);
1220 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
1221 case MVT::v16f16: return fastEmit_ISD_FSQRT_MVT_v16f16_r(RetVT, Op0);
1222 case MVT::v32f16: return fastEmit_ISD_FSQRT_MVT_v32f16_r(RetVT, Op0);
1223 case MVT::v8bf16: return fastEmit_ISD_FSQRT_MVT_v8bf16_r(RetVT, Op0);
1224 case MVT::v16bf16: return fastEmit_ISD_FSQRT_MVT_v16bf16_r(RetVT, Op0);
1225 case MVT::v32bf16: return fastEmit_ISD_FSQRT_MVT_v32bf16_r(RetVT, Op0);
1226 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
1227 case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0);
1228 case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0);
1229 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
1230 case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0);
1231 case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0);
1232 default: return Register();
1233 }
1234}
1235
1236// FastEmit functions for ISD::LLRINT.
1237
1238Register fastEmit_ISD_LLRINT_MVT_f16_r(MVT RetVT, Register Op0) {
1239 if (RetVT.SimpleTy != MVT::i64)
1240 return Register();
1241 if ((Subtarget->hasFP16()) && (Subtarget->is64Bit())) {
1242 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1243 }
1244 return Register();
1245}
1246
1247Register fastEmit_ISD_LLRINT_MVT_f32_r(MVT RetVT, Register Op0) {
1248 if (RetVT.SimpleTy != MVT::i64)
1249 return Register();
1250 if ((Subtarget->hasAVX512())) {
1251 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1252 }
1253 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1254 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1255 }
1256 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1257 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1258 }
1259 return Register();
1260}
1261
1262Register fastEmit_ISD_LLRINT_MVT_f64_r(MVT RetVT, Register Op0) {
1263 if (RetVT.SimpleTy != MVT::i64)
1264 return Register();
1265 if ((Subtarget->hasAVX512())) {
1266 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1267 }
1268 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1269 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1270 }
1271 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1272 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1273 }
1274 return Register();
1275}
1276
1277Register fastEmit_ISD_LLRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1278 if (RetVT.SimpleTy != MVT::v8i64)
1279 return Register();
1280 if ((Subtarget->hasFP16())) {
1281 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
1282 }
1283 return Register();
1284}
1285
1286Register fastEmit_ISD_LLRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1287 if (RetVT.SimpleTy != MVT::v4i64)
1288 return Register();
1289 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1290 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1291 }
1292 return Register();
1293}
1294
1295Register fastEmit_ISD_LLRINT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1296 if (RetVT.SimpleTy != MVT::v8i64)
1297 return Register();
1298 if ((Subtarget->hasDQI())) {
1299 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1300 }
1301 return Register();
1302}
1303
1304Register fastEmit_ISD_LLRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1305 if (RetVT.SimpleTy != MVT::v2i64)
1306 return Register();
1307 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1308 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1309 }
1310 return Register();
1311}
1312
1313Register fastEmit_ISD_LLRINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1314 if (RetVT.SimpleTy != MVT::v4i64)
1315 return Register();
1316 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1317 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1318 }
1319 return Register();
1320}
1321
1322Register fastEmit_ISD_LLRINT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1323 if (RetVT.SimpleTy != MVT::v8i64)
1324 return Register();
1325 if ((Subtarget->hasDQI())) {
1326 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1327 }
1328 return Register();
1329}
1330
1331Register fastEmit_ISD_LLRINT_r(MVT VT, MVT RetVT, Register Op0) {
1332 switch (VT.SimpleTy) {
1333 case MVT::f16: return fastEmit_ISD_LLRINT_MVT_f16_r(RetVT, Op0);
1334 case MVT::f32: return fastEmit_ISD_LLRINT_MVT_f32_r(RetVT, Op0);
1335 case MVT::f64: return fastEmit_ISD_LLRINT_MVT_f64_r(RetVT, Op0);
1336 case MVT::v8f16: return fastEmit_ISD_LLRINT_MVT_v8f16_r(RetVT, Op0);
1337 case MVT::v4f32: return fastEmit_ISD_LLRINT_MVT_v4f32_r(RetVT, Op0);
1338 case MVT::v8f32: return fastEmit_ISD_LLRINT_MVT_v8f32_r(RetVT, Op0);
1339 case MVT::v2f64: return fastEmit_ISD_LLRINT_MVT_v2f64_r(RetVT, Op0);
1340 case MVT::v4f64: return fastEmit_ISD_LLRINT_MVT_v4f64_r(RetVT, Op0);
1341 case MVT::v8f64: return fastEmit_ISD_LLRINT_MVT_v8f64_r(RetVT, Op0);
1342 default: return Register();
1343 }
1344}
1345
1346// FastEmit functions for ISD::LRINT.
1347
1348Register fastEmit_ISD_LRINT_MVT_f16_MVT_i32_r(Register Op0) {
1349 if ((Subtarget->hasFP16())) {
1350 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
1351 }
1352 return Register();
1353}
1354
1355Register fastEmit_ISD_LRINT_MVT_f16_MVT_i64_r(Register Op0) {
1356 if ((Subtarget->hasFP16()) && (Subtarget->is64Bit())) {
1357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1358 }
1359 return Register();
1360}
1361
1362Register fastEmit_ISD_LRINT_MVT_f16_r(MVT RetVT, Register Op0) {
1363switch (RetVT.SimpleTy) {
1364 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f16_MVT_i32_r(Op0);
1365 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f16_MVT_i64_r(Op0);
1366 default: return Register();
1367}
1368}
1369
1370Register fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Register Op0) {
1371 if ((Subtarget->hasAVX512())) {
1372 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
1373 }
1374 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1375 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1376 }
1377 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1378 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1379 }
1380 return Register();
1381}
1382
1383Register fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Register Op0) {
1384 if ((Subtarget->hasAVX512())) {
1385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1386 }
1387 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1388 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1389 }
1390 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1391 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1392 }
1393 return Register();
1394}
1395
1396Register fastEmit_ISD_LRINT_MVT_f32_r(MVT RetVT, Register Op0) {
1397switch (RetVT.SimpleTy) {
1398 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Op0);
1399 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Op0);
1400 default: return Register();
1401}
1402}
1403
1404Register fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Register Op0) {
1405 if ((Subtarget->hasAVX512())) {
1406 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
1407 }
1408 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1409 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1410 }
1411 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1412 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1413 }
1414 return Register();
1415}
1416
1417Register fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Register Op0) {
1418 if ((Subtarget->hasAVX512())) {
1419 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1420 }
1421 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1422 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1423 }
1424 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1425 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1426 }
1427 return Register();
1428}
1429
1430Register fastEmit_ISD_LRINT_MVT_f64_r(MVT RetVT, Register Op0) {
1431switch (RetVT.SimpleTy) {
1432 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Op0);
1433 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Op0);
1434 default: return Register();
1435}
1436}
1437
1438Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i16_r(Register Op0) {
1439 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1440 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
1441 }
1442 return Register();
1443}
1444
1445Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i32_r(Register Op0) {
1446 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1447 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1448 }
1449 return Register();
1450}
1451
1452Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i64_r(Register Op0) {
1453 if ((Subtarget->hasFP16())) {
1454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
1455 }
1456 return Register();
1457}
1458
1459Register fastEmit_ISD_LRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1460switch (RetVT.SimpleTy) {
1461 case MVT::v8i16: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i16_r(Op0);
1462 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i32_r(Op0);
1463 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i64_r(Op0);
1464 default: return Register();
1465}
1466}
1467
1468Register fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i16_r(Register Op0) {
1469 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1470 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
1471 }
1472 return Register();
1473}
1474
1475Register fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i32_r(Register Op0) {
1476 if ((Subtarget->hasFP16())) {
1477 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
1478 }
1479 return Register();
1480}
1481
1482Register fastEmit_ISD_LRINT_MVT_v16f16_r(MVT RetVT, Register Op0) {
1483switch (RetVT.SimpleTy) {
1484 case MVT::v16i16: return fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i16_r(Op0);
1485 case MVT::v16i32: return fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i32_r(Op0);
1486 default: return Register();
1487}
1488}
1489
1490Register fastEmit_ISD_LRINT_MVT_v32f16_r(MVT RetVT, Register Op0) {
1491 if (RetVT.SimpleTy != MVT::v32i16)
1492 return Register();
1493 if ((Subtarget->hasFP16())) {
1494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
1495 }
1496 return Register();
1497}
1498
1499Register fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Register Op0) {
1500 if ((Subtarget->hasVLX())) {
1501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
1502 }
1503 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1504 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1505 }
1506 if ((Subtarget->hasAVX())) {
1507 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1508 }
1509 return Register();
1510}
1511
1512Register fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Register Op0) {
1513 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1514 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1515 }
1516 return Register();
1517}
1518
1519Register fastEmit_ISD_LRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1520switch (RetVT.SimpleTy) {
1521 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Op0);
1522 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Op0);
1523 default: return Register();
1524}
1525}
1526
1527Register fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Register Op0) {
1528 if ((Subtarget->hasVLX())) {
1529 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1530 }
1531 if ((Subtarget->hasAVX())) {
1532 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
1533 }
1534 return Register();
1535}
1536
1537Register fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Register Op0) {
1538 if ((Subtarget->hasDQI())) {
1539 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1540 }
1541 return Register();
1542}
1543
1544Register fastEmit_ISD_LRINT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1545switch (RetVT.SimpleTy) {
1546 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Op0);
1547 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Op0);
1548 default: return Register();
1549}
1550}
1551
1552Register fastEmit_ISD_LRINT_MVT_v16f32_r(MVT RetVT, Register Op0) {
1553 if (RetVT.SimpleTy != MVT::v16i32)
1554 return Register();
1555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
1556}
1557
1558Register fastEmit_ISD_LRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1559 if (RetVT.SimpleTy != MVT::v2i64)
1560 return Register();
1561 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1562 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1563 }
1564 return Register();
1565}
1566
1567Register fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Register Op0) {
1568 if ((Subtarget->hasVLX())) {
1569 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
1570 }
1571 if ((Subtarget->hasAVX())) {
1572 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
1573 }
1574 return Register();
1575}
1576
1577Register fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Register Op0) {
1578 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1579 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1580 }
1581 return Register();
1582}
1583
1584Register fastEmit_ISD_LRINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1585switch (RetVT.SimpleTy) {
1586 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Op0);
1587 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Op0);
1588 default: return Register();
1589}
1590}
1591
1592Register fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Register Op0) {
1593 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
1594}
1595
1596Register fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Register Op0) {
1597 if ((Subtarget->hasDQI())) {
1598 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1599 }
1600 return Register();
1601}
1602
1603Register fastEmit_ISD_LRINT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1604switch (RetVT.SimpleTy) {
1605 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Op0);
1606 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Op0);
1607 default: return Register();
1608}
1609}
1610
1611Register fastEmit_ISD_LRINT_r(MVT VT, MVT RetVT, Register Op0) {
1612 switch (VT.SimpleTy) {
1613 case MVT::f16: return fastEmit_ISD_LRINT_MVT_f16_r(RetVT, Op0);
1614 case MVT::f32: return fastEmit_ISD_LRINT_MVT_f32_r(RetVT, Op0);
1615 case MVT::f64: return fastEmit_ISD_LRINT_MVT_f64_r(RetVT, Op0);
1616 case MVT::v8f16: return fastEmit_ISD_LRINT_MVT_v8f16_r(RetVT, Op0);
1617 case MVT::v16f16: return fastEmit_ISD_LRINT_MVT_v16f16_r(RetVT, Op0);
1618 case MVT::v32f16: return fastEmit_ISD_LRINT_MVT_v32f16_r(RetVT, Op0);
1619 case MVT::v4f32: return fastEmit_ISD_LRINT_MVT_v4f32_r(RetVT, Op0);
1620 case MVT::v8f32: return fastEmit_ISD_LRINT_MVT_v8f32_r(RetVT, Op0);
1621 case MVT::v16f32: return fastEmit_ISD_LRINT_MVT_v16f32_r(RetVT, Op0);
1622 case MVT::v2f64: return fastEmit_ISD_LRINT_MVT_v2f64_r(RetVT, Op0);
1623 case MVT::v4f64: return fastEmit_ISD_LRINT_MVT_v4f64_r(RetVT, Op0);
1624 case MVT::v8f64: return fastEmit_ISD_LRINT_MVT_v8f64_r(RetVT, Op0);
1625 default: return Register();
1626 }
1627}
1628
1629// FastEmit functions for ISD::SCALAR_TO_VECTOR.
1630
1631Register fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
1632 if (RetVT.SimpleTy != MVT::v4i32)
1633 return Register();
1634 if ((Subtarget->hasAVX512())) {
1635 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIZrr, RC: &X86::VR128XRegClass, Op0);
1636 }
1637 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1638 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1639 }
1640 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1641 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1642 }
1643 return Register();
1644}
1645
1646Register fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
1647 if (RetVT.SimpleTy != MVT::v2i64)
1648 return Register();
1649 if ((Subtarget->hasAVX512())) {
1650 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIZrr, RC: &X86::VR128XRegClass, Op0);
1651 }
1652 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1653 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1654 }
1655 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1656 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1657 }
1658 return Register();
1659}
1660
1661Register fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
1662 switch (VT.SimpleTy) {
1663 case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0);
1664 case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0);
1665 default: return Register();
1666 }
1667}
1668
1669// FastEmit functions for ISD::SIGN_EXTEND.
1670
1671Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Register Op0) {
1672 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr8, RC: &X86::GR32RegClass, Op0);
1673}
1674
1675Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Register Op0) {
1676 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr8, RC: &X86::GR64RegClass, Op0);
1677}
1678
1679Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
1680switch (RetVT.SimpleTy) {
1681 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0);
1682 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0);
1683 default: return Register();
1684}
1685}
1686
1687Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Register Op0) {
1688 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr16, RC: &X86::GR32RegClass, Op0);
1689}
1690
1691Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Register Op0) {
1692 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr16, RC: &X86::GR64RegClass, Op0);
1693}
1694
1695Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, Register Op0) {
1696switch (RetVT.SimpleTy) {
1697 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0);
1698 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0);
1699 default: return Register();
1700}
1701}
1702
1703Register fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, Register Op0) {
1704 if (RetVT.SimpleTy != MVT::i64)
1705 return Register();
1706 if ((Subtarget->is64Bit())) {
1707 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr32, RC: &X86::GR64RegClass, Op0);
1708 }
1709 return Register();
1710}
1711
1712Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, Register Op0) {
1713 if (RetVT.SimpleTy != MVT::v2i64)
1714 return Register();
1715 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1716 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ128rk, RC: &X86::VR128XRegClass, Op0);
1717 }
1718 return Register();
1719}
1720
1721Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Register Op0) {
1722 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1723 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ128rk, RC: &X86::VR128XRegClass, Op0);
1724 }
1725 return Register();
1726}
1727
1728Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Register Op0) {
1729 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1730 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ256rk, RC: &X86::VR256XRegClass, Op0);
1731 }
1732 return Register();
1733}
1734
1735Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, Register Op0) {
1736switch (RetVT.SimpleTy) {
1737 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0);
1738 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0);
1739 default: return Register();
1740}
1741}
1742
1743Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Register Op0) {
1744 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1745 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ128rk, RC: &X86::VR128XRegClass, Op0);
1746 }
1747 return Register();
1748}
1749
1750Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Register Op0) {
1751 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1752 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ256rk, RC: &X86::VR256XRegClass, Op0);
1753 }
1754 return Register();
1755}
1756
1757Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Register Op0) {
1758 if ((Subtarget->hasDQI())) {
1759 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZrk, RC: &X86::VR512RegClass, Op0);
1760 }
1761 return Register();
1762}
1763
1764Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, Register Op0) {
1765switch (RetVT.SimpleTy) {
1766 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0);
1767 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0);
1768 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0);
1769 default: return Register();
1770}
1771}
1772
1773Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Register Op0) {
1774 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1775 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ128rk, RC: &X86::VR128XRegClass, Op0);
1776 }
1777 return Register();
1778}
1779
1780Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Register Op0) {
1781 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1782 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ256rk, RC: &X86::VR256XRegClass, Op0);
1783 }
1784 return Register();
1785}
1786
1787Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Register Op0) {
1788 if ((Subtarget->hasDQI())) {
1789 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZrk, RC: &X86::VR512RegClass, Op0);
1790 }
1791 return Register();
1792}
1793
1794Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, Register Op0) {
1795switch (RetVT.SimpleTy) {
1796 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0);
1797 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0);
1798 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0);
1799 default: return Register();
1800}
1801}
1802
1803Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Register Op0) {
1804 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1805 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ256rk, RC: &X86::VR256XRegClass, Op0);
1806 }
1807 return Register();
1808}
1809
1810Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Register Op0) {
1811 if ((Subtarget->hasBWI())) {
1812 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZrk, RC: &X86::VR512RegClass, Op0);
1813 }
1814 return Register();
1815}
1816
1817Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, Register Op0) {
1818switch (RetVT.SimpleTy) {
1819 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0);
1820 case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0);
1821 default: return Register();
1822}
1823}
1824
1825Register fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, Register Op0) {
1826 if (RetVT.SimpleTy != MVT::v64i8)
1827 return Register();
1828 if ((Subtarget->hasBWI())) {
1829 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZrk, RC: &X86::VR512RegClass, Op0);
1830 }
1831 return Register();
1832}
1833
1834Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Register Op0) {
1835 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1836 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWYrr, RC: &X86::VR256RegClass, Op0);
1837 }
1838 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1839 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
1840 }
1841 return Register();
1842}
1843
1844Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Register Op0) {
1845 if ((Subtarget->hasAVX512())) {
1846 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZrr, RC: &X86::VR512RegClass, Op0);
1847 }
1848 return Register();
1849}
1850
1851Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(MVT RetVT, Register Op0) {
1852switch (RetVT.SimpleTy) {
1853 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
1854 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
1855 default: return Register();
1856}
1857}
1858
1859Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(MVT RetVT, Register Op0) {
1860 if (RetVT.SimpleTy != MVT::v32i16)
1861 return Register();
1862 if ((Subtarget->hasBWI())) {
1863 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZrr, RC: &X86::VR512RegClass, Op0);
1864 }
1865 return Register();
1866}
1867
1868Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Register Op0) {
1869 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1870 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDYrr, RC: &X86::VR256RegClass, Op0);
1871 }
1872 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1873 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
1874 }
1875 return Register();
1876}
1877
1878Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Register Op0) {
1879 if ((Subtarget->hasAVX512())) {
1880 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZrr, RC: &X86::VR512RegClass, Op0);
1881 }
1882 return Register();
1883}
1884
1885Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(MVT RetVT, Register Op0) {
1886switch (RetVT.SimpleTy) {
1887 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
1888 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
1889 default: return Register();
1890}
1891}
1892
1893Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(MVT RetVT, Register Op0) {
1894 if (RetVT.SimpleTy != MVT::v16i32)
1895 return Register();
1896 if ((Subtarget->hasAVX512())) {
1897 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZrr, RC: &X86::VR512RegClass, Op0);
1898 }
1899 return Register();
1900}
1901
1902Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(MVT RetVT, Register Op0) {
1903 if (RetVT.SimpleTy != MVT::v4i64)
1904 return Register();
1905 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1906 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQYrr, RC: &X86::VR256RegClass, Op0);
1907 }
1908 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1909 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
1910 }
1911 return Register();
1912}
1913
1914Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(MVT RetVT, Register Op0) {
1915 if (RetVT.SimpleTy != MVT::v8i64)
1916 return Register();
1917 if ((Subtarget->hasAVX512())) {
1918 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZrr, RC: &X86::VR512RegClass, Op0);
1919 }
1920 return Register();
1921}
1922
1923Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
1924 switch (VT.SimpleTy) {
1925 case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0);
1926 case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0);
1927 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0);
1928 case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0);
1929 case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0);
1930 case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0);
1931 case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0);
1932 case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0);
1933 case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0);
1934 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(RetVT, Op0);
1935 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(RetVT, Op0);
1936 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(RetVT, Op0);
1937 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(RetVT, Op0);
1938 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(RetVT, Op0);
1939 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(RetVT, Op0);
1940 default: return Register();
1941 }
1942}
1943
1944// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
1945
1946Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1947 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1948 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1949 }
1950 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1951 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1952 }
1953 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1954 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
1955 }
1956 return Register();
1957}
1958
1959Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1960 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1961 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
1962 }
1963 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1964 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
1965 }
1966 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1967 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
1968 }
1969 return Register();
1970}
1971
1972Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Register Op0) {
1973 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1974 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDYrr, RC: &X86::VR256RegClass, Op0);
1975 }
1976 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1977 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
1978 }
1979 return Register();
1980}
1981
1982Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1983 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1984 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
1985 }
1986 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1987 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
1988 }
1989 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1990 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
1991 }
1992 return Register();
1993}
1994
1995Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Register Op0) {
1996 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1997 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQYrr, RC: &X86::VR256RegClass, Op0);
1998 }
1999 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2000 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
2001 }
2002 return Register();
2003}
2004
2005Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Register Op0) {
2006 if ((Subtarget->hasAVX512())) {
2007 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZrr, RC: &X86::VR512RegClass, Op0);
2008 }
2009 return Register();
2010}
2011
2012Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, Register Op0) {
2013switch (RetVT.SimpleTy) {
2014 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
2015 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
2016 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
2017 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
2018 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
2019 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
2020 default: return Register();
2021}
2022}
2023
2024Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Register Op0) {
2025 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2026 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
2027 }
2028 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2029 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
2030 }
2031 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2032 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
2033 }
2034 return Register();
2035}
2036
2037Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Register Op0) {
2038 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2039 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
2040 }
2041 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2042 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
2043 }
2044 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2045 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
2046 }
2047 return Register();
2048}
2049
2050Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Register Op0) {
2051 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2052 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQYrr, RC: &X86::VR256RegClass, Op0);
2053 }
2054 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2055 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
2056 }
2057 return Register();
2058}
2059
2060Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, Register Op0) {
2061switch (RetVT.SimpleTy) {
2062 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
2063 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
2064 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
2065 default: return Register();
2066}
2067}
2068
2069Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, Register Op0) {
2070 if (RetVT.SimpleTy != MVT::v2i64)
2071 return Register();
2072 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2073 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
2074 }
2075 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2076 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
2077 }
2078 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2079 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
2080 }
2081 return Register();
2082}
2083
2084Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, Register Op0) {
2085 switch (VT.SimpleTy) {
2086 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
2087 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
2088 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
2089 default: return Register();
2090 }
2091}
2092
2093// FastEmit functions for ISD::SINT_TO_FP.
2094
2095Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
2096 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2097 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2098 }
2099 return Register();
2100}
2101
2102Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
2103 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2104 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2105 }
2106 return Register();
2107}
2108
2109Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
2110switch (RetVT.SimpleTy) {
2111 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2112 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2113 default: return Register();
2114}
2115}
2116
2117Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
2118 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2119 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2120 }
2121 return Register();
2122}
2123
2124Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
2125 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2126 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2127 }
2128 return Register();
2129}
2130
2131Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
2132switch (RetVT.SimpleTy) {
2133 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2134 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2135 default: return Register();
2136}
2137}
2138
2139Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2140 if (RetVT.SimpleTy != MVT::v8f16)
2141 return Register();
2142 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2144 }
2145 return Register();
2146}
2147
2148Register fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
2149 if (RetVT.SimpleTy != MVT::v16f16)
2150 return Register();
2151 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2152 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2153 }
2154 return Register();
2155}
2156
2157Register fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
2158 if (RetVT.SimpleTy != MVT::v32f16)
2159 return Register();
2160 if ((Subtarget->hasFP16())) {
2161 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2162 }
2163 return Register();
2164}
2165
2166Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
2167 if ((Subtarget->hasVLX())) {
2168 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2169 }
2170 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2171 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2172 }
2173 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2174 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2175 }
2176 return Register();
2177}
2178
2179Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
2180 if ((Subtarget->hasVLX())) {
2181 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2182 }
2183 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2184 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2185 }
2186 return Register();
2187}
2188
2189Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2190switch (RetVT.SimpleTy) {
2191 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2192 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2193 default: return Register();
2194}
2195}
2196
2197Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
2198 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2199 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2200 }
2201 return Register();
2202}
2203
2204Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
2205 if ((Subtarget->hasVLX())) {
2206 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2207 }
2208 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2210 }
2211 return Register();
2212}
2213
2214Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
2215 if ((Subtarget->hasAVX512())) {
2216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2217 }
2218 return Register();
2219}
2220
2221Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
2222switch (RetVT.SimpleTy) {
2223 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2224 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2225 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2226 default: return Register();
2227}
2228}
2229
2230Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
2231 if ((Subtarget->hasFP16())) {
2232 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2233 }
2234 return Register();
2235}
2236
2237Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
2238 if ((Subtarget->hasAVX512())) {
2239 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2240 }
2241 return Register();
2242}
2243
2244Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
2245switch (RetVT.SimpleTy) {
2246 case MVT::v16f16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2247 case MVT::v16f32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2248 default: return Register();
2249}
2250}
2251
2252Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2253 if (RetVT.SimpleTy != MVT::v2f64)
2254 return Register();
2255 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2256 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2257 }
2258 return Register();
2259}
2260
2261Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
2262 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2263 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2264 }
2265 return Register();
2266}
2267
2268Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
2269 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2270 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2271 }
2272 return Register();
2273}
2274
2275Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
2276switch (RetVT.SimpleTy) {
2277 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2278 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2279 default: return Register();
2280}
2281}
2282
2283Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
2284 if ((Subtarget->hasFP16())) {
2285 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2286 }
2287 return Register();
2288}
2289
2290Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
2291 if ((Subtarget->hasDQI())) {
2292 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2293 }
2294 return Register();
2295}
2296
2297Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
2298 if ((Subtarget->hasDQI())) {
2299 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2300 }
2301 return Register();
2302}
2303
2304Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
2305switch (RetVT.SimpleTy) {
2306 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
2307 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
2308 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
2309 default: return Register();
2310}
2311}
2312
2313Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
2314 switch (VT.SimpleTy) {
2315 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
2316 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
2317 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2318 case MVT::v16i16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
2319 case MVT::v32i16: return fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
2320 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2321 case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
2322 case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
2323 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
2324 case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
2325 case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
2326 default: return Register();
2327 }
2328}
2329
2330// FastEmit functions for ISD::STRICT_FP_EXTEND.
2331
2332Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2333 if (RetVT.SimpleTy != MVT::f64)
2334 return Register();
2335 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2336 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
2337 }
2338 return Register();
2339}
2340
2341Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Register Op0) {
2342 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2343 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
2344 }
2345 return Register();
2346}
2347
2348Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Register Op0) {
2349 if ((Subtarget->hasFP16())) {
2350 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
2351 }
2352 return Register();
2353}
2354
2355Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2356switch (RetVT.SimpleTy) {
2357 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
2358 case MVT::v8f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
2359 default: return Register();
2360}
2361}
2362
2363Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(MVT RetVT, Register Op0) {
2364 if (RetVT.SimpleTy != MVT::v16f32)
2365 return Register();
2366 if ((Subtarget->hasFP16())) {
2367 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
2368 }
2369 return Register();
2370}
2371
2372Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2373 if (RetVT.SimpleTy != MVT::v4f64)
2374 return Register();
2375 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2376 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2377 }
2378 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2379 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
2380 }
2381 return Register();
2382}
2383
2384Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(MVT RetVT, Register Op0) {
2385 if (RetVT.SimpleTy != MVT::v8f64)
2386 return Register();
2387 if ((Subtarget->hasAVX512())) {
2388 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
2389 }
2390 return Register();
2391}
2392
2393Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2394 switch (VT.SimpleTy) {
2395 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2396 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
2397 case MVT::v16f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
2398 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
2399 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
2400 default: return Register();
2401 }
2402}
2403
2404// FastEmit functions for ISD::STRICT_FP_ROUND.
2405
2406Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2407 if (RetVT.SimpleTy != MVT::f32)
2408 return Register();
2409 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2410 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
2411 }
2412 return Register();
2413}
2414
2415Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2416 switch (VT.SimpleTy) {
2417 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
2418 default: return Register();
2419 }
2420}
2421
2422// FastEmit functions for ISD::STRICT_FP_TO_SINT.
2423
2424Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
2425 if ((Subtarget->hasFP16())) {
2426 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
2427 }
2428 return Register();
2429}
2430
2431Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
2432 if ((Subtarget->hasFP16())) {
2433 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2434 }
2435 return Register();
2436}
2437
2438Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
2439switch (RetVT.SimpleTy) {
2440 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
2441 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
2442 default: return Register();
2443}
2444}
2445
2446Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
2447 if ((Subtarget->hasAVX512())) {
2448 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
2449 }
2450 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2451 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2452 }
2453 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2455 }
2456 return Register();
2457}
2458
2459Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
2460 if ((Subtarget->hasAVX512())) {
2461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2462 }
2463 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2464 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2465 }
2466 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2467 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2468 }
2469 return Register();
2470}
2471
2472Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
2473switch (RetVT.SimpleTy) {
2474 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
2475 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
2476 default: return Register();
2477}
2478}
2479
2480Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
2481 if ((Subtarget->hasAVX512())) {
2482 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
2483 }
2484 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2485 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2486 }
2487 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2488 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2489 }
2490 return Register();
2491}
2492
2493Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
2494 if ((Subtarget->hasAVX512())) {
2495 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2496 }
2497 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2498 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2499 }
2500 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2502 }
2503 return Register();
2504}
2505
2506Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
2507switch (RetVT.SimpleTy) {
2508 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
2509 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
2510 default: return Register();
2511}
2512}
2513
2514Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
2515 if (RetVT.SimpleTy != MVT::v4i32)
2516 return Register();
2517 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2518 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
2519 }
2520 return Register();
2521}
2522
2523Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2524 switch (VT.SimpleTy) {
2525 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
2526 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
2527 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
2528 case MVT::v4f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
2529 default: return Register();
2530 }
2531}
2532
2533// FastEmit functions for ISD::STRICT_FP_TO_UINT.
2534
2535Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
2536 if ((Subtarget->hasFP16())) {
2537 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
2538 }
2539 return Register();
2540}
2541
2542Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
2543 if ((Subtarget->hasFP16())) {
2544 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2545 }
2546 return Register();
2547}
2548
2549Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
2550switch (RetVT.SimpleTy) {
2551 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
2552 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
2553 default: return Register();
2554}
2555}
2556
2557Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
2558 if ((Subtarget->hasAVX512())) {
2559 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
2560 }
2561 return Register();
2562}
2563
2564Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
2565 if ((Subtarget->hasAVX512())) {
2566 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2567 }
2568 return Register();
2569}
2570
2571Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
2572switch (RetVT.SimpleTy) {
2573 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
2574 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
2575 default: return Register();
2576}
2577}
2578
2579Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
2580 if ((Subtarget->hasAVX512())) {
2581 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
2582 }
2583 return Register();
2584}
2585
2586Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
2587 if ((Subtarget->hasAVX512())) {
2588 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2589 }
2590 return Register();
2591}
2592
2593Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
2594switch (RetVT.SimpleTy) {
2595 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
2596 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
2597 default: return Register();
2598}
2599}
2600
2601Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2602 switch (VT.SimpleTy) {
2603 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
2604 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
2605 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
2606 default: return Register();
2607 }
2608}
2609
2610// FastEmit functions for ISD::STRICT_FSQRT.
2611
2612Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
2613 if (RetVT.SimpleTy != MVT::f32)
2614 return Register();
2615 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2616 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
2617 }
2618 if ((!Subtarget->hasSSE1())) {
2619 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
2620 }
2621 return Register();
2622}
2623
2624Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
2625 if (RetVT.SimpleTy != MVT::f64)
2626 return Register();
2627 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2628 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
2629 }
2630 if ((!Subtarget->hasSSE2())) {
2631 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
2632 }
2633 return Register();
2634}
2635
2636Register fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(MVT RetVT, Register Op0) {
2637 if (RetVT.SimpleTy != MVT::f80)
2638 return Register();
2639 if ((Subtarget->hasX87())) {
2640 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
2641 }
2642 return Register();
2643}
2644
2645Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2646 if (RetVT.SimpleTy != MVT::v8f16)
2647 return Register();
2648 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2649 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
2650 }
2651 return Register();
2652}
2653
2654Register fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(MVT RetVT, Register Op0) {
2655 if (RetVT.SimpleTy != MVT::v16f16)
2656 return Register();
2657 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2658 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
2659 }
2660 return Register();
2661}
2662
2663Register fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(MVT RetVT, Register Op0) {
2664 if (RetVT.SimpleTy != MVT::v32f16)
2665 return Register();
2666 if ((Subtarget->hasFP16())) {
2667 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
2668 }
2669 return Register();
2670}
2671
2672Register fastEmit_ISD_STRICT_FSQRT_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2673 if (RetVT.SimpleTy != MVT::v8bf16)
2674 return Register();
2675 if ((Subtarget->hasAVX10_2())) {
2676 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
2677 }
2678 return Register();
2679}
2680
2681Register fastEmit_ISD_STRICT_FSQRT_MVT_v16bf16_r(MVT RetVT, Register Op0) {
2682 if (RetVT.SimpleTy != MVT::v16bf16)
2683 return Register();
2684 if ((Subtarget->hasAVX10_2())) {
2685 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
2686 }
2687 return Register();
2688}
2689
2690Register fastEmit_ISD_STRICT_FSQRT_MVT_v32bf16_r(MVT RetVT, Register Op0) {
2691 if (RetVT.SimpleTy != MVT::v32bf16)
2692 return Register();
2693 if ((Subtarget->hasAVX10_2())) {
2694 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
2695 }
2696 return Register();
2697}
2698
2699Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2700 if (RetVT.SimpleTy != MVT::v4f32)
2701 return Register();
2702 if ((Subtarget->hasVLX())) {
2703 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
2704 }
2705 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2706 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
2707 }
2708 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2709 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
2710 }
2711 return Register();
2712}
2713
2714Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
2715 if (RetVT.SimpleTy != MVT::v8f32)
2716 return Register();
2717 if ((Subtarget->hasVLX())) {
2718 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
2719 }
2720 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2721 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
2722 }
2723 return Register();
2724}
2725
2726Register fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(MVT RetVT, Register Op0) {
2727 if (RetVT.SimpleTy != MVT::v16f32)
2728 return Register();
2729 if ((Subtarget->hasAVX512())) {
2730 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
2731 }
2732 return Register();
2733}
2734
2735Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2736 if (RetVT.SimpleTy != MVT::v2f64)
2737 return Register();
2738 if ((Subtarget->hasVLX())) {
2739 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
2740 }
2741 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2742 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
2743 }
2744 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2745 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
2746 }
2747 return Register();
2748}
2749
2750Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(MVT RetVT, Register Op0) {
2751 if (RetVT.SimpleTy != MVT::v4f64)
2752 return Register();
2753 if ((Subtarget->hasVLX())) {
2754 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
2755 }
2756 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2757 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
2758 }
2759 return Register();
2760}
2761
2762Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(MVT RetVT, Register Op0) {
2763 if (RetVT.SimpleTy != MVT::v8f64)
2764 return Register();
2765 if ((Subtarget->hasAVX512())) {
2766 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
2767 }
2768 return Register();
2769}
2770
2771Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
2772 switch (VT.SimpleTy) {
2773 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
2774 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
2775 case MVT::f80: return fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(RetVT, Op0);
2776 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
2777 case MVT::v16f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(RetVT, Op0);
2778 case MVT::v32f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(RetVT, Op0);
2779 case MVT::v8bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8bf16_r(RetVT, Op0);
2780 case MVT::v16bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16bf16_r(RetVT, Op0);
2781 case MVT::v32bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32bf16_r(RetVT, Op0);
2782 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
2783 case MVT::v8f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(RetVT, Op0);
2784 case MVT::v16f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(RetVT, Op0);
2785 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
2786 case MVT::v4f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(RetVT, Op0);
2787 case MVT::v8f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(RetVT, Op0);
2788 default: return Register();
2789 }
2790}
2791
2792// FastEmit functions for ISD::STRICT_SINT_TO_FP.
2793
2794Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
2795 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2796 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2797 }
2798 return Register();
2799}
2800
2801Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
2802 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2803 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2804 }
2805 return Register();
2806}
2807
2808Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
2809switch (RetVT.SimpleTy) {
2810 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2811 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2812 default: return Register();
2813}
2814}
2815
2816Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
2817 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2818 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2819 }
2820 return Register();
2821}
2822
2823Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
2824 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2825 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2826 }
2827 return Register();
2828}
2829
2830Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
2831switch (RetVT.SimpleTy) {
2832 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2833 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2834 default: return Register();
2835}
2836}
2837
2838Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2839 if (RetVT.SimpleTy != MVT::v8f16)
2840 return Register();
2841 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2842 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2843 }
2844 return Register();
2845}
2846
2847Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
2848 if (RetVT.SimpleTy != MVT::v16f16)
2849 return Register();
2850 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2851 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2852 }
2853 return Register();
2854}
2855
2856Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
2857 if (RetVT.SimpleTy != MVT::v32f16)
2858 return Register();
2859 if ((Subtarget->hasFP16())) {
2860 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2861 }
2862 return Register();
2863}
2864
2865Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
2866 if ((Subtarget->hasVLX())) {
2867 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2868 }
2869 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2870 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2871 }
2872 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2873 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2874 }
2875 return Register();
2876}
2877
2878Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
2879 if ((Subtarget->hasVLX())) {
2880 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2881 }
2882 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2883 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2884 }
2885 return Register();
2886}
2887
2888Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2889switch (RetVT.SimpleTy) {
2890 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2891 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2892 default: return Register();
2893}
2894}
2895
2896Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
2897 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2898 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2899 }
2900 return Register();
2901}
2902
2903Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
2904 if ((Subtarget->hasVLX())) {
2905 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2906 }
2907 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2908 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2909 }
2910 return Register();
2911}
2912
2913Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
2914 if ((Subtarget->hasAVX512())) {
2915 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2916 }
2917 return Register();
2918}
2919
2920Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
2921switch (RetVT.SimpleTy) {
2922 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2923 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2924 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2925 default: return Register();
2926}
2927}
2928
2929Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
2930 if ((Subtarget->hasFP16())) {
2931 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2932 }
2933 return Register();
2934}
2935
2936Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
2937 if ((Subtarget->hasAVX512())) {
2938 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2939 }
2940 return Register();
2941}
2942
2943Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
2944switch (RetVT.SimpleTy) {
2945 case MVT::v16f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2946 case MVT::v16f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2947 default: return Register();
2948}
2949}
2950
2951Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2952 if (RetVT.SimpleTy != MVT::v2f64)
2953 return Register();
2954 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2955 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2956 }
2957 return Register();
2958}
2959
2960Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
2961 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2963 }
2964 return Register();
2965}
2966
2967Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
2968 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2969 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2970 }
2971 return Register();
2972}
2973
2974Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
2975switch (RetVT.SimpleTy) {
2976 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2977 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2978 default: return Register();
2979}
2980}
2981
2982Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
2983 if ((Subtarget->hasFP16())) {
2984 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2985 }
2986 return Register();
2987}
2988
2989Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
2990 if ((Subtarget->hasDQI())) {
2991 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2992 }
2993 return Register();
2994}
2995
2996Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
2997 if ((Subtarget->hasDQI())) {
2998 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2999 }
3000 return Register();
3001}
3002
3003Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3004switch (RetVT.SimpleTy) {
3005 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3006 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3007 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3008 default: return Register();
3009}
3010}
3011
3012Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3013 switch (VT.SimpleTy) {
3014 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
3015 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
3016 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3017 case MVT::v16i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3018 case MVT::v32i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3019 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3020 case MVT::v8i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3021 case MVT::v16i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3022 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3023 case MVT::v4i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3024 case MVT::v8i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3025 default: return Register();
3026 }
3027}
3028
3029// FastEmit functions for ISD::STRICT_UINT_TO_FP.
3030
3031Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3032 if (RetVT.SimpleTy != MVT::v8f16)
3033 return Register();
3034 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3035 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3036 }
3037 return Register();
3038}
3039
3040Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
3041 if (RetVT.SimpleTy != MVT::v16f16)
3042 return Register();
3043 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3044 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3045 }
3046 return Register();
3047}
3048
3049Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
3050 if (RetVT.SimpleTy != MVT::v32f16)
3051 return Register();
3052 if ((Subtarget->hasFP16())) {
3053 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3054 }
3055 return Register();
3056}
3057
3058Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
3059 if ((Subtarget->hasVLX())) {
3060 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3061 }
3062 return Register();
3063}
3064
3065Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
3066 if ((Subtarget->hasVLX())) {
3067 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3068 }
3069 return Register();
3070}
3071
3072Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3073switch (RetVT.SimpleTy) {
3074 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3075 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3076 default: return Register();
3077}
3078}
3079
3080Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
3081 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3082 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3083 }
3084 return Register();
3085}
3086
3087Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
3088 if ((Subtarget->hasVLX())) {
3089 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3090 }
3091 return Register();
3092}
3093
3094Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
3095 if ((Subtarget->hasAVX512())) {
3096 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3097 }
3098 return Register();
3099}
3100
3101Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
3102switch (RetVT.SimpleTy) {
3103 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3104 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3105 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3106 default: return Register();
3107}
3108}
3109
3110Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
3111 if ((Subtarget->hasFP16())) {
3112 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3113 }
3114 return Register();
3115}
3116
3117Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
3118 if ((Subtarget->hasAVX512())) {
3119 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3120 }
3121 return Register();
3122}
3123
3124Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
3125switch (RetVT.SimpleTy) {
3126 case MVT::v16f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3127 case MVT::v16f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3128 default: return Register();
3129}
3130}
3131
3132Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
3133 if (RetVT.SimpleTy != MVT::v2f64)
3134 return Register();
3135 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3136 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3137 }
3138 return Register();
3139}
3140
3141Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
3142 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3143 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3144 }
3145 return Register();
3146}
3147
3148Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3149 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3150 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3151 }
3152 return Register();
3153}
3154
3155Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3156switch (RetVT.SimpleTy) {
3157 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3158 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3159 default: return Register();
3160}
3161}
3162
3163Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3164 if ((Subtarget->hasFP16())) {
3165 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3166 }
3167 return Register();
3168}
3169
3170Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3171 if ((Subtarget->hasDQI())) {
3172 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3173 }
3174 return Register();
3175}
3176
3177Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3178 if ((Subtarget->hasDQI())) {
3179 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3180 }
3181 return Register();
3182}
3183
3184Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3185switch (RetVT.SimpleTy) {
3186 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3187 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3188 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3189 default: return Register();
3190}
3191}
3192
3193Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3194 switch (VT.SimpleTy) {
3195 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3196 case MVT::v16i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3197 case MVT::v32i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3198 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3199 case MVT::v8i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3200 case MVT::v16i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3201 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3202 case MVT::v4i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3203 case MVT::v8i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3204 default: return Register();
3205 }
3206}
3207
3208// FastEmit functions for ISD::TRUNCATE.
3209
3210Register fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, Register Op0) {
3211 if (RetVT.SimpleTy != MVT::i8)
3212 return Register();
3213 if ((Subtarget->is64Bit())) {
3214 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: X86::sub_8bit);
3215 }
3216 return Register();
3217}
3218
3219Register fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Register Op0) {
3220 if ((Subtarget->is64Bit())) {
3221 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3222 }
3223 return Register();
3224}
3225
3226Register fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Register Op0) {
3227 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3228}
3229
3230Register fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, Register Op0) {
3231switch (RetVT.SimpleTy) {
3232 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0);
3233 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0);
3234 default: return Register();
3235}
3236}
3237
3238Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Register Op0) {
3239 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3240}
3241
3242Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Register Op0) {
3243 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3244}
3245
3246Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Register Op0) {
3247 return fastEmitInst_extractsubreg(RetVT: MVT::i32, Op0, Idx: X86::sub_32bit);
3248}
3249
3250Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
3251switch (RetVT.SimpleTy) {
3252 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0);
3253 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0);
3254 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0);
3255 default: return Register();
3256}
3257}
3258
3259Register fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, Register Op0) {
3260 if (RetVT.SimpleTy != MVT::v16i8)
3261 return Register();
3262 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3263 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ256rr, RC: &X86::VR128XRegClass, Op0);
3264 }
3265 return Register();
3266}
3267
3268Register fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, Register Op0) {
3269 if (RetVT.SimpleTy != MVT::v32i8)
3270 return Register();
3271 if ((Subtarget->hasBWI())) {
3272 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZrr, RC: &X86::VR256XRegClass, Op0);
3273 }
3274 return Register();
3275}
3276
3277Register fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, Register Op0) {
3278 if (RetVT.SimpleTy != MVT::v8i16)
3279 return Register();
3280 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3281 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ256rr, RC: &X86::VR128XRegClass, Op0);
3282 }
3283 return Register();
3284}
3285
3286Register fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Register Op0) {
3287 if ((Subtarget->hasAVX512())) {
3288 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZrr, RC: &X86::VR128XRegClass, Op0);
3289 }
3290 return Register();
3291}
3292
3293Register fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Register Op0) {
3294 if ((Subtarget->hasAVX512())) {
3295 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZrr, RC: &X86::VR256XRegClass, Op0);
3296 }
3297 return Register();
3298}
3299
3300Register fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, Register Op0) {
3301switch (RetVT.SimpleTy) {
3302 case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0);
3303 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0);
3304 default: return Register();
3305}
3306}
3307
3308Register fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, Register Op0) {
3309 if (RetVT.SimpleTy != MVT::v4i32)
3310 return Register();
3311 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3312 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ256rr, RC: &X86::VR128XRegClass, Op0);
3313 }
3314 return Register();
3315}
3316
3317Register fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Register Op0) {
3318 if ((Subtarget->hasAVX512())) {
3319 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZrr, RC: &X86::VR128XRegClass, Op0);
3320 }
3321 return Register();
3322}
3323
3324Register fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Register Op0) {
3325 if ((Subtarget->hasAVX512())) {
3326 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZrr, RC: &X86::VR256XRegClass, Op0);
3327 }
3328 return Register();
3329}
3330
3331Register fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, Register Op0) {
3332switch (RetVT.SimpleTy) {
3333 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0);
3334 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0);
3335 default: return Register();
3336}
3337}
3338
3339Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3340 switch (VT.SimpleTy) {
3341 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0);
3342 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0);
3343 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
3344 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0);
3345 case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0);
3346 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0);
3347 case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0);
3348 case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0);
3349 case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0);
3350 default: return Register();
3351 }
3352}
3353
3354// FastEmit functions for ISD::UINT_TO_FP.
3355
3356Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3357 if (RetVT.SimpleTy != MVT::v8f16)
3358 return Register();
3359 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3360 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3361 }
3362 return Register();
3363}
3364
3365Register fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
3366 if (RetVT.SimpleTy != MVT::v16f16)
3367 return Register();
3368 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3369 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3370 }
3371 return Register();
3372}
3373
3374Register fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
3375 if (RetVT.SimpleTy != MVT::v32f16)
3376 return Register();
3377 if ((Subtarget->hasFP16())) {
3378 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3379 }
3380 return Register();
3381}
3382
3383Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
3384 if ((Subtarget->hasVLX())) {
3385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3386 }
3387 return Register();
3388}
3389
3390Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
3391 if ((Subtarget->hasVLX())) {
3392 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3393 }
3394 return Register();
3395}
3396
3397Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3398switch (RetVT.SimpleTy) {
3399 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3400 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3401 default: return Register();
3402}
3403}
3404
3405Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
3406 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3407 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3408 }
3409 return Register();
3410}
3411
3412Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
3413 if ((Subtarget->hasVLX())) {
3414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3415 }
3416 return Register();
3417}
3418
3419Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
3420 if ((Subtarget->hasAVX512())) {
3421 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3422 }
3423 return Register();
3424}
3425
3426Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
3427switch (RetVT.SimpleTy) {
3428 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3429 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3430 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3431 default: return Register();
3432}
3433}
3434
3435Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
3436 if ((Subtarget->hasFP16())) {
3437 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3438 }
3439 return Register();
3440}
3441
3442Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
3443 if ((Subtarget->hasAVX512())) {
3444 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3445 }
3446 return Register();
3447}
3448
3449Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
3450switch (RetVT.SimpleTy) {
3451 case MVT::v16f16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3452 case MVT::v16f32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3453 default: return Register();
3454}
3455}
3456
3457Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
3458 if (RetVT.SimpleTy != MVT::v2f64)
3459 return Register();
3460 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3462 }
3463 return Register();
3464}
3465
3466Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
3467 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3469 }
3470 return Register();
3471}
3472
3473Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3474 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3475 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3476 }
3477 return Register();
3478}
3479
3480Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3481switch (RetVT.SimpleTy) {
3482 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3483 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3484 default: return Register();
3485}
3486}
3487
3488Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3489 if ((Subtarget->hasFP16())) {
3490 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3491 }
3492 return Register();
3493}
3494
3495Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3496 if ((Subtarget->hasDQI())) {
3497 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3498 }
3499 return Register();
3500}
3501
3502Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3503 if ((Subtarget->hasDQI())) {
3504 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3505 }
3506 return Register();
3507}
3508
3509Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3510switch (RetVT.SimpleTy) {
3511 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3512 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3513 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3514 default: return Register();
3515}
3516}
3517
3518Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3519 switch (VT.SimpleTy) {
3520 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3521 case MVT::v16i16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3522 case MVT::v32i16: return fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3523 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3524 case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3525 case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3526 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3527 case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3528 case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3529 default: return Register();
3530 }
3531}
3532
3533// FastEmit functions for ISD::ZERO_EXTEND.
3534
3535Register fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
3536 if (RetVT.SimpleTy != MVT::i32)
3537 return Register();
3538 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
3539}
3540
3541Register fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, Register Op0) {
3542 if (RetVT.SimpleTy != MVT::i32)
3543 return Register();
3544 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr16, RC: &X86::GR32RegClass, Op0);
3545}
3546
3547Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Register Op0) {
3548 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3549 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWYrr, RC: &X86::VR256RegClass, Op0);
3550 }
3551 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3552 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
3553 }
3554 return Register();
3555}
3556
3557Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Register Op0) {
3558 if ((Subtarget->hasAVX512())) {
3559 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZrr, RC: &X86::VR512RegClass, Op0);
3560 }
3561 return Register();
3562}
3563
3564Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(MVT RetVT, Register Op0) {
3565switch (RetVT.SimpleTy) {
3566 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
3567 case MVT::v16i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
3568 default: return Register();
3569}
3570}
3571
3572Register fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(MVT RetVT, Register Op0) {
3573 if (RetVT.SimpleTy != MVT::v32i16)
3574 return Register();
3575 if ((Subtarget->hasBWI())) {
3576 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZrr, RC: &X86::VR512RegClass, Op0);
3577 }
3578 return Register();
3579}
3580
3581Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Register Op0) {
3582 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3583 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDYrr, RC: &X86::VR256RegClass, Op0);
3584 }
3585 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3586 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
3587 }
3588 return Register();
3589}
3590
3591Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Register Op0) {
3592 if ((Subtarget->hasAVX512())) {
3593 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZrr, RC: &X86::VR512RegClass, Op0);
3594 }
3595 return Register();
3596}
3597
3598Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(MVT RetVT, Register Op0) {
3599switch (RetVT.SimpleTy) {
3600 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
3601 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
3602 default: return Register();
3603}
3604}
3605
3606Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(MVT RetVT, Register Op0) {
3607 if (RetVT.SimpleTy != MVT::v16i32)
3608 return Register();
3609 if ((Subtarget->hasAVX512())) {
3610 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZrr, RC: &X86::VR512RegClass, Op0);
3611 }
3612 return Register();
3613}
3614
3615Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(MVT RetVT, Register Op0) {
3616 if (RetVT.SimpleTy != MVT::v4i64)
3617 return Register();
3618 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3619 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQYrr, RC: &X86::VR256RegClass, Op0);
3620 }
3621 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3622 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
3623 }
3624 return Register();
3625}
3626
3627Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(MVT RetVT, Register Op0) {
3628 if (RetVT.SimpleTy != MVT::v8i64)
3629 return Register();
3630 if ((Subtarget->hasAVX512())) {
3631 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZrr, RC: &X86::VR512RegClass, Op0);
3632 }
3633 return Register();
3634}
3635
3636Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3637 switch (VT.SimpleTy) {
3638 case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0);
3639 case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0);
3640 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(RetVT, Op0);
3641 case MVT::v32i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(RetVT, Op0);
3642 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(RetVT, Op0);
3643 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(RetVT, Op0);
3644 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(RetVT, Op0);
3645 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(RetVT, Op0);
3646 default: return Register();
3647 }
3648}
3649
3650// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
3651
3652Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Register Op0) {
3653 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3654 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3655 }
3656 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3657 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3658 }
3659 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3660 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
3661 }
3662 return Register();
3663}
3664
3665Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Register Op0) {
3666 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3667 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3668 }
3669 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3670 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3671 }
3672 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3673 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
3674 }
3675 return Register();
3676}
3677
3678Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Register Op0) {
3679 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3680 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDYrr, RC: &X86::VR256RegClass, Op0);
3681 }
3682 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3683 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
3684 }
3685 return Register();
3686}
3687
3688Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Register Op0) {
3689 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3690 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3691 }
3692 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3693 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3694 }
3695 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3696 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
3697 }
3698 return Register();
3699}
3700
3701Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Register Op0) {
3702 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3703 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQYrr, RC: &X86::VR256RegClass, Op0);
3704 }
3705 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3706 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
3707 }
3708 return Register();
3709}
3710
3711Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Register Op0) {
3712 if ((Subtarget->hasAVX512())) {
3713 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZrr, RC: &X86::VR512RegClass, Op0);
3714 }
3715 return Register();
3716}
3717
3718Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, Register Op0) {
3719switch (RetVT.SimpleTy) {
3720 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
3721 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
3722 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
3723 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
3724 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
3725 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
3726 default: return Register();
3727}
3728}
3729
3730Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Register Op0) {
3731 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3732 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3733 }
3734 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3735 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3736 }
3737 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3738 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
3739 }
3740 return Register();
3741}
3742
3743Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Register Op0) {
3744 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3745 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3746 }
3747 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3748 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3749 }
3750 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3751 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
3752 }
3753 return Register();
3754}
3755
3756Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Register Op0) {
3757 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3758 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQYrr, RC: &X86::VR256RegClass, Op0);
3759 }
3760 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3761 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
3762 }
3763 return Register();
3764}
3765
3766Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, Register Op0) {
3767switch (RetVT.SimpleTy) {
3768 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
3769 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
3770 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
3771 default: return Register();
3772}
3773}
3774
3775Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, Register Op0) {
3776 if (RetVT.SimpleTy != MVT::v2i64)
3777 return Register();
3778 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3779 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3780 }
3781 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3782 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3783 }
3784 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3785 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
3786 }
3787 return Register();
3788}
3789
3790Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, Register Op0) {
3791 switch (VT.SimpleTy) {
3792 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
3793 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
3794 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
3795 default: return Register();
3796 }
3797}
3798
3799// FastEmit functions for X86ISD::CALL.
3800
3801Register fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, Register Op0) {
3802 if (RetVT.SimpleTy != MVT::isVoid)
3803 return Register();
3804 if ((!Subtarget->is64Bit())) {
3805 return fastEmitInst_r(MachineInstOpcode: X86::CALL16r, RC: &X86::GR16RegClass, Op0);
3806 }
3807 return Register();
3808}
3809
3810Register fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
3811 if (RetVT.SimpleTy != MVT::isVoid)
3812 return Register();
3813 if ((!Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3814 return fastEmitInst_r(MachineInstOpcode: X86::INDIRECT_THUNK_CALL32, RC: &X86::GR32RegClass, Op0);
3815 }
3816 if ((!Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3817 return fastEmitInst_r(MachineInstOpcode: X86::CALL32r, RC: &X86::GR32RegClass, Op0);
3818 }
3819 return Register();
3820}
3821
3822Register fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
3823 if (RetVT.SimpleTy != MVT::isVoid)
3824 return Register();
3825 if ((MF->getFunction().getParent()->getModuleFlag(Key: "import-call-optimization")) && (Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3826 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r_ImpCall, RC: &X86::GR64_ARegClass, Op0);
3827 }
3828 if ((Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3829 return fastEmitInst_r(MachineInstOpcode: X86::INDIRECT_THUNK_CALL64, RC: &X86::GR64RegClass, Op0);
3830 }
3831 if ((!MF->getFunction().getParent()->getModuleFlag(Key: "import-call-optimization")) && (Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3832 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r, RC: &X86::GR64RegClass, Op0);
3833 }
3834 return Register();
3835}
3836
3837Register fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
3838 switch (VT.SimpleTy) {
3839 case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0);
3840 case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0);
3841 case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0);
3842 default: return Register();
3843 }
3844}
3845
3846// FastEmit functions for X86ISD::CONFLICT.
3847
3848Register fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, Register Op0) {
3849 if (RetVT.SimpleTy != MVT::v4i32)
3850 return Register();
3851 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3852 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ128rr, RC: &X86::VR128XRegClass, Op0);
3853 }
3854 return Register();
3855}
3856
3857Register fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, Register Op0) {
3858 if (RetVT.SimpleTy != MVT::v8i32)
3859 return Register();
3860 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3861 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ256rr, RC: &X86::VR256XRegClass, Op0);
3862 }
3863 return Register();
3864}
3865
3866Register fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, Register Op0) {
3867 if (RetVT.SimpleTy != MVT::v16i32)
3868 return Register();
3869 if ((Subtarget->hasCDI())) {
3870 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZrr, RC: &X86::VR512RegClass, Op0);
3871 }
3872 return Register();
3873}
3874
3875Register fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, Register Op0) {
3876 if (RetVT.SimpleTy != MVT::v2i64)
3877 return Register();
3878 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3879 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ128rr, RC: &X86::VR128XRegClass, Op0);
3880 }
3881 return Register();
3882}
3883
3884Register fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, Register Op0) {
3885 if (RetVT.SimpleTy != MVT::v4i64)
3886 return Register();
3887 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3888 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ256rr, RC: &X86::VR256XRegClass, Op0);
3889 }
3890 return Register();
3891}
3892
3893Register fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, Register Op0) {
3894 if (RetVT.SimpleTy != MVT::v8i64)
3895 return Register();
3896 if ((Subtarget->hasCDI())) {
3897 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZrr, RC: &X86::VR512RegClass, Op0);
3898 }
3899 return Register();
3900}
3901
3902Register fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, Register Op0) {
3903 switch (VT.SimpleTy) {
3904 case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0);
3905 case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0);
3906 case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0);
3907 case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0);
3908 case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0);
3909 case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0);
3910 default: return Register();
3911 }
3912}
3913
3914// FastEmit functions for X86ISD::CVTNEPS2BF16.
3915
3916Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(MVT RetVT, Register Op0) {
3917 if (RetVT.SimpleTy != MVT::v8bf16)
3918 return Register();
3919 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3920 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0);
3921 }
3922 if ((Subtarget->hasAVXNECONVERT())) {
3923 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16rr, RC: &X86::VR128RegClass, Op0);
3924 }
3925 return Register();
3926}
3927
3928Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(MVT RetVT, Register Op0) {
3929 if (RetVT.SimpleTy != MVT::v8bf16)
3930 return Register();
3931 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3932 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
3933 }
3934 return Register();
3935}
3936
3937Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(MVT RetVT, Register Op0) {
3938 if (RetVT.SimpleTy != MVT::v16bf16)
3939 return Register();
3940 if ((Subtarget->hasBF16())) {
3941 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
3942 }
3943 return Register();
3944}
3945
3946Register fastEmit_X86ISD_CVTNEPS2BF16_r(MVT VT, MVT RetVT, Register Op0) {
3947 switch (VT.SimpleTy) {
3948 case MVT::v4f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(RetVT, Op0);
3949 case MVT::v8f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(RetVT, Op0);
3950 case MVT::v16f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(RetVT, Op0);
3951 default: return Register();
3952 }
3953}
3954
3955// FastEmit functions for X86ISD::CVTP2IBS.
3956
3957Register fastEmit_X86ISD_CVTP2IBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
3958 if (RetVT.SimpleTy != MVT::v8i16)
3959 return Register();
3960 if ((Subtarget->hasAVX10_2())) {
3961 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
3962 }
3963 return Register();
3964}
3965
3966Register fastEmit_X86ISD_CVTP2IBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
3967 if (RetVT.SimpleTy != MVT::v16i16)
3968 return Register();
3969 if ((Subtarget->hasAVX10_2())) {
3970 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
3971 }
3972 return Register();
3973}
3974
3975Register fastEmit_X86ISD_CVTP2IBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
3976 if (RetVT.SimpleTy != MVT::v32i16)
3977 return Register();
3978 if ((Subtarget->hasAVX10_2())) {
3979 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZrr, RC: &X86::VR512RegClass, Op0);
3980 }
3981 return Register();
3982}
3983
3984Register fastEmit_X86ISD_CVTP2IBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
3985 if (RetVT.SimpleTy != MVT::v8i16)
3986 return Register();
3987 if ((Subtarget->hasAVX10_2())) {
3988 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
3989 }
3990 return Register();
3991}
3992
3993Register fastEmit_X86ISD_CVTP2IBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
3994 if (RetVT.SimpleTy != MVT::v16i16)
3995 return Register();
3996 if ((Subtarget->hasAVX10_2())) {
3997 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
3998 }
3999 return Register();
4000}
4001
4002Register fastEmit_X86ISD_CVTP2IBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4003 if (RetVT.SimpleTy != MVT::v32i16)
4004 return Register();
4005 if ((Subtarget->hasAVX10_2())) {
4006 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZrr, RC: &X86::VR512RegClass, Op0);
4007 }
4008 return Register();
4009}
4010
4011Register fastEmit_X86ISD_CVTP2IBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4012 if (RetVT.SimpleTy != MVT::v4i32)
4013 return Register();
4014 if ((Subtarget->hasAVX10_2())) {
4015 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4016 }
4017 return Register();
4018}
4019
4020Register fastEmit_X86ISD_CVTP2IBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
4021 if (RetVT.SimpleTy != MVT::v8i32)
4022 return Register();
4023 if ((Subtarget->hasAVX10_2())) {
4024 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4025 }
4026 return Register();
4027}
4028
4029Register fastEmit_X86ISD_CVTP2IBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
4030 if (RetVT.SimpleTy != MVT::v16i32)
4031 return Register();
4032 if ((Subtarget->hasAVX10_2())) {
4033 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZrr, RC: &X86::VR512RegClass, Op0);
4034 }
4035 return Register();
4036}
4037
4038Register fastEmit_X86ISD_CVTP2IBS_r(MVT VT, MVT RetVT, Register Op0) {
4039 switch (VT.SimpleTy) {
4040 case MVT::v8f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v8f16_r(RetVT, Op0);
4041 case MVT::v16f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v16f16_r(RetVT, Op0);
4042 case MVT::v32f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v32f16_r(RetVT, Op0);
4043 case MVT::v8bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v8bf16_r(RetVT, Op0);
4044 case MVT::v16bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v16bf16_r(RetVT, Op0);
4045 case MVT::v32bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v32bf16_r(RetVT, Op0);
4046 case MVT::v4f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v4f32_r(RetVT, Op0);
4047 case MVT::v8f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v8f32_r(RetVT, Op0);
4048 case MVT::v16f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v16f32_r(RetVT, Op0);
4049 default: return Register();
4050 }
4051}
4052
4053// FastEmit functions for X86ISD::CVTP2IUBS.
4054
4055Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
4056 if (RetVT.SimpleTy != MVT::v8i16)
4057 return Register();
4058 if ((Subtarget->hasAVX10_2())) {
4059 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4060 }
4061 return Register();
4062}
4063
4064Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4065 if (RetVT.SimpleTy != MVT::v16i16)
4066 return Register();
4067 if ((Subtarget->hasAVX10_2())) {
4068 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4069 }
4070 return Register();
4071}
4072
4073Register fastEmit_X86ISD_CVTP2IUBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4074 if (RetVT.SimpleTy != MVT::v32i16)
4075 return Register();
4076 if ((Subtarget->hasAVX10_2())) {
4077 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZrr, RC: &X86::VR512RegClass, Op0);
4078 }
4079 return Register();
4080}
4081
4082Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
4083 if (RetVT.SimpleTy != MVT::v8i16)
4084 return Register();
4085 if ((Subtarget->hasAVX10_2())) {
4086 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4087 }
4088 return Register();
4089}
4090
4091Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
4092 if (RetVT.SimpleTy != MVT::v16i16)
4093 return Register();
4094 if ((Subtarget->hasAVX10_2())) {
4095 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4096 }
4097 return Register();
4098}
4099
4100Register fastEmit_X86ISD_CVTP2IUBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4101 if (RetVT.SimpleTy != MVT::v32i16)
4102 return Register();
4103 if ((Subtarget->hasAVX10_2())) {
4104 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZrr, RC: &X86::VR512RegClass, Op0);
4105 }
4106 return Register();
4107}
4108
4109Register fastEmit_X86ISD_CVTP2IUBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4110 if (RetVT.SimpleTy != MVT::v4i32)
4111 return Register();
4112 if ((Subtarget->hasAVX10_2())) {
4113 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4114 }
4115 return Register();
4116}
4117
4118Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
4119 if (RetVT.SimpleTy != MVT::v8i32)
4120 return Register();
4121 if ((Subtarget->hasAVX10_2())) {
4122 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4123 }
4124 return Register();
4125}
4126
4127Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
4128 if (RetVT.SimpleTy != MVT::v16i32)
4129 return Register();
4130 if ((Subtarget->hasAVX10_2())) {
4131 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZrr, RC: &X86::VR512RegClass, Op0);
4132 }
4133 return Register();
4134}
4135
4136Register fastEmit_X86ISD_CVTP2IUBS_r(MVT VT, MVT RetVT, Register Op0) {
4137 switch (VT.SimpleTy) {
4138 case MVT::v8f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8f16_r(RetVT, Op0);
4139 case MVT::v16f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16f16_r(RetVT, Op0);
4140 case MVT::v32f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v32f16_r(RetVT, Op0);
4141 case MVT::v8bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8bf16_r(RetVT, Op0);
4142 case MVT::v16bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16bf16_r(RetVT, Op0);
4143 case MVT::v32bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v32bf16_r(RetVT, Op0);
4144 case MVT::v4f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v4f32_r(RetVT, Op0);
4145 case MVT::v8f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8f32_r(RetVT, Op0);
4146 case MVT::v16f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16f32_r(RetVT, Op0);
4147 default: return Register();
4148 }
4149}
4150
4151// FastEmit functions for X86ISD::CVTP2SI.
4152
4153Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
4154 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4155 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
4156 }
4157 return Register();
4158}
4159
4160Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
4161 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4162 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4163 }
4164 return Register();
4165}
4166
4167Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
4168 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4169 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4170 }
4171 return Register();
4172}
4173
4174Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
4175 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4176 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4177 }
4178 return Register();
4179}
4180
4181Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
4182 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4183 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4184 }
4185 return Register();
4186}
4187
4188Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
4189 if ((Subtarget->hasFP16())) {
4190 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
4191 }
4192 return Register();
4193}
4194
4195Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4196switch (RetVT.SimpleTy) {
4197 case MVT::v8i16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
4198 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
4199 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
4200 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
4201 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
4202 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
4203 default: return Register();
4204}
4205}
4206
4207Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
4208 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
4210 }
4211 return Register();
4212}
4213
4214Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
4215 if ((Subtarget->hasFP16())) {
4216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
4217 }
4218 return Register();
4219}
4220
4221Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
4222switch (RetVT.SimpleTy) {
4223 case MVT::v16i16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
4224 case MVT::v16i32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
4225 default: return Register();
4226}
4227}
4228
4229Register fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
4230 if (RetVT.SimpleTy != MVT::v32i16)
4231 return Register();
4232 if ((Subtarget->hasFP16())) {
4233 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
4234 }
4235 return Register();
4236}
4237
4238Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
4239 if ((Subtarget->hasVLX())) {
4240 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4241 }
4242 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4243 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4244 }
4245 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4246 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4247 }
4248 return Register();
4249}
4250
4251Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
4252 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4253 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4254 }
4255 return Register();
4256}
4257
4258Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
4259 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4260 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4261 }
4262 return Register();
4263}
4264
4265Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4266switch (RetVT.SimpleTy) {
4267 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
4268 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
4269 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
4270 default: return Register();
4271}
4272}
4273
4274Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
4275 if ((Subtarget->hasVLX())) {
4276 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4277 }
4278 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4279 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
4280 }
4281 return Register();
4282}
4283
4284Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
4285 if ((Subtarget->hasDQI())) {
4286 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
4287 }
4288 return Register();
4289}
4290
4291Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
4292switch (RetVT.SimpleTy) {
4293 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
4294 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
4295 default: return Register();
4296}
4297}
4298
4299Register fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
4300 if (RetVT.SimpleTy != MVT::v16i32)
4301 return Register();
4302 if ((Subtarget->hasAVX512())) {
4303 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
4304 }
4305 return Register();
4306}
4307
4308Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
4309 if ((Subtarget->hasVLX())) {
4310 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4311 }
4312 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4313 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4314 }
4315 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4316 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4317 }
4318 return Register();
4319}
4320
4321Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
4322 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4323 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4324 }
4325 return Register();
4326}
4327
4328Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4329switch (RetVT.SimpleTy) {
4330 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
4331 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
4332 default: return Register();
4333}
4334}
4335
4336Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
4337 if ((Subtarget->hasVLX())) {
4338 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
4339 }
4340 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4341 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
4342 }
4343 return Register();
4344}
4345
4346Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
4347 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4349 }
4350 return Register();
4351}
4352
4353Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
4354switch (RetVT.SimpleTy) {
4355 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
4356 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
4357 default: return Register();
4358}
4359}
4360
4361Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
4362 if ((Subtarget->hasAVX512())) {
4363 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
4364 }
4365 return Register();
4366}
4367
4368Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
4369 if ((Subtarget->hasDQI())) {
4370 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
4371 }
4372 return Register();
4373}
4374
4375Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
4376switch (RetVT.SimpleTy) {
4377 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
4378 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
4379 default: return Register();
4380}
4381}
4382
4383Register fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
4384 switch (VT.SimpleTy) {
4385 case MVT::v8f16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(RetVT, Op0);
4386 case MVT::v16f16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(RetVT, Op0);
4387 case MVT::v32f16: return fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(RetVT, Op0);
4388 case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0);
4389 case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0);
4390 case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0);
4391 case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0);
4392 case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0);
4393 case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0);
4394 default: return Register();
4395 }
4396}
4397
4398// FastEmit functions for X86ISD::CVTP2UI.
4399
4400Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
4401 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4402 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
4403 }
4404 return Register();
4405}
4406
4407Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
4408 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4409 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4410 }
4411 return Register();
4412}
4413
4414Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
4415 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4416 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4417 }
4418 return Register();
4419}
4420
4421Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
4422 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4423 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4424 }
4425 return Register();
4426}
4427
4428Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
4429 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4430 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4431 }
4432 return Register();
4433}
4434
4435Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
4436 if ((Subtarget->hasFP16())) {
4437 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
4438 }
4439 return Register();
4440}
4441
4442Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4443switch (RetVT.SimpleTy) {
4444 case MVT::v8i16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
4445 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
4446 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
4447 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
4448 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
4449 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
4450 default: return Register();
4451}
4452}
4453
4454Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
4455 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4456 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
4457 }
4458 return Register();
4459}
4460
4461Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
4462 if ((Subtarget->hasFP16())) {
4463 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
4464 }
4465 return Register();
4466}
4467
4468Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
4469switch (RetVT.SimpleTy) {
4470 case MVT::v16i16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
4471 case MVT::v16i32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
4472 default: return Register();
4473}
4474}
4475
4476Register fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
4477 if (RetVT.SimpleTy != MVT::v32i16)
4478 return Register();
4479 if ((Subtarget->hasFP16())) {
4480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
4481 }
4482 return Register();
4483}
4484
4485Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
4486 if ((Subtarget->hasVLX())) {
4487 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4488 }
4489 return Register();
4490}
4491
4492Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
4493 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4495 }
4496 return Register();
4497}
4498
4499Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
4500 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4502 }
4503 return Register();
4504}
4505
4506Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4507switch (RetVT.SimpleTy) {
4508 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
4509 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
4510 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
4511 default: return Register();
4512}
4513}
4514
4515Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
4516 if ((Subtarget->hasVLX())) {
4517 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4518 }
4519 return Register();
4520}
4521
4522Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
4523 if ((Subtarget->hasDQI())) {
4524 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
4525 }
4526 return Register();
4527}
4528
4529Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
4530switch (RetVT.SimpleTy) {
4531 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
4532 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
4533 default: return Register();
4534}
4535}
4536
4537Register fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
4538 if (RetVT.SimpleTy != MVT::v16i32)
4539 return Register();
4540 if ((Subtarget->hasAVX512())) {
4541 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
4542 }
4543 return Register();
4544}
4545
4546Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
4547 if ((Subtarget->hasVLX())) {
4548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4549 }
4550 return Register();
4551}
4552
4553Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
4554 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4556 }
4557 return Register();
4558}
4559
4560Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4561switch (RetVT.SimpleTy) {
4562 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
4563 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
4564 default: return Register();
4565}
4566}
4567
4568Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
4569 if ((Subtarget->hasVLX())) {
4570 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
4571 }
4572 return Register();
4573}
4574
4575Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
4576 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4578 }
4579 return Register();
4580}
4581
4582Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
4583switch (RetVT.SimpleTy) {
4584 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
4585 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
4586 default: return Register();
4587}
4588}
4589
4590Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
4591 if ((Subtarget->hasAVX512())) {
4592 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
4593 }
4594 return Register();
4595}
4596
4597Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
4598 if ((Subtarget->hasDQI())) {
4599 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
4600 }
4601 return Register();
4602}
4603
4604Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
4605switch (RetVT.SimpleTy) {
4606 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
4607 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
4608 default: return Register();
4609}
4610}
4611
4612Register fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
4613 switch (VT.SimpleTy) {
4614 case MVT::v8f16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(RetVT, Op0);
4615 case MVT::v16f16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(RetVT, Op0);
4616 case MVT::v32f16: return fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(RetVT, Op0);
4617 case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0);
4618 case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0);
4619 case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0);
4620 case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0);
4621 case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0);
4622 case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0);
4623 default: return Register();
4624 }
4625}
4626
4627// FastEmit functions for X86ISD::CVTPH2PS.
4628
4629Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Register Op0) {
4630 if ((Subtarget->hasVLX())) {
4631 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4632 }
4633 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4634 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
4635 }
4636 return Register();
4637}
4638
4639Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Register Op0) {
4640 if ((Subtarget->hasVLX())) {
4641 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
4642 }
4643 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4644 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
4645 }
4646 return Register();
4647}
4648
4649Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, Register Op0) {
4650switch (RetVT.SimpleTy) {
4651 case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
4652 case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
4653 default: return Register();
4654}
4655}
4656
4657Register fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, Register Op0) {
4658 if (RetVT.SimpleTy != MVT::v16f32)
4659 return Register();
4660 if ((Subtarget->hasAVX512())) {
4661 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
4662 }
4663 return Register();
4664}
4665
4666Register fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, Register Op0) {
4667 switch (VT.SimpleTy) {
4668 case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
4669 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
4670 default: return Register();
4671 }
4672}
4673
4674// FastEmit functions for X86ISD::CVTPH2PS_SAE.
4675
4676Register fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(MVT RetVT, Register Op0) {
4677 if (RetVT.SimpleTy != MVT::v16f32)
4678 return Register();
4679 if ((Subtarget->hasAVX512())) {
4680 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrrb, RC: &X86::VR512RegClass, Op0);
4681 }
4682 return Register();
4683}
4684
4685Register fastEmit_X86ISD_CVTPH2PS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
4686 switch (VT.SimpleTy) {
4687 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(RetVT, Op0);
4688 default: return Register();
4689 }
4690}
4691
4692// FastEmit functions for X86ISD::CVTS2SI.
4693
4694Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Register Op0) {
4695 if ((Subtarget->hasFP16())) {
4696 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4697 }
4698 return Register();
4699}
4700
4701Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Register Op0) {
4702 if ((Subtarget->hasFP16())) {
4703 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4704 }
4705 return Register();
4706}
4707
4708Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4709switch (RetVT.SimpleTy) {
4710 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Op0);
4711 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Op0);
4712 default: return Register();
4713}
4714}
4715
4716Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Register Op0) {
4717 if ((Subtarget->hasAVX512())) {
4718 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4719 }
4720 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4721 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4722 }
4723 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4724 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4725 }
4726 return Register();
4727}
4728
4729Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Register Op0) {
4730 if ((Subtarget->hasAVX512())) {
4731 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4732 }
4733 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4734 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4735 }
4736 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4737 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4738 }
4739 return Register();
4740}
4741
4742Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4743switch (RetVT.SimpleTy) {
4744 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0);
4745 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0);
4746 default: return Register();
4747}
4748}
4749
4750Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Register Op0) {
4751 if ((Subtarget->hasAVX512())) {
4752 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4753 }
4754 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4755 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4756 }
4757 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4758 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4759 }
4760 return Register();
4761}
4762
4763Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Register Op0) {
4764 if ((Subtarget->hasAVX512())) {
4765 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4766 }
4767 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4768 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4769 }
4770 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4771 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4772 }
4773 return Register();
4774}
4775
4776Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4777switch (RetVT.SimpleTy) {
4778 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0);
4779 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0);
4780 default: return Register();
4781}
4782}
4783
4784Register fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, Register Op0) {
4785 switch (VT.SimpleTy) {
4786 case MVT::v8f16: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(RetVT, Op0);
4787 case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0);
4788 case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0);
4789 default: return Register();
4790 }
4791}
4792
4793// FastEmit functions for X86ISD::CVTS2UI.
4794
4795Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Register Op0) {
4796 if ((Subtarget->hasFP16())) {
4797 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4798 }
4799 return Register();
4800}
4801
4802Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Register Op0) {
4803 if ((Subtarget->hasFP16())) {
4804 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4805 }
4806 return Register();
4807}
4808
4809Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4810switch (RetVT.SimpleTy) {
4811 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Op0);
4812 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Op0);
4813 default: return Register();
4814}
4815}
4816
4817Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Register Op0) {
4818 if ((Subtarget->hasAVX512())) {
4819 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4820 }
4821 return Register();
4822}
4823
4824Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Register Op0) {
4825 if ((Subtarget->hasAVX512())) {
4826 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4827 }
4828 return Register();
4829}
4830
4831Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4832switch (RetVT.SimpleTy) {
4833 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0);
4834 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0);
4835 default: return Register();
4836}
4837}
4838
4839Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Register Op0) {
4840 if ((Subtarget->hasAVX512())) {
4841 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4842 }
4843 return Register();
4844}
4845
4846Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Register Op0) {
4847 if ((Subtarget->hasAVX512())) {
4848 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4849 }
4850 return Register();
4851}
4852
4853Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4854switch (RetVT.SimpleTy) {
4855 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0);
4856 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0);
4857 default: return Register();
4858}
4859}
4860
4861Register fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, Register Op0) {
4862 switch (VT.SimpleTy) {
4863 case MVT::v8f16: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(RetVT, Op0);
4864 case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0);
4865 case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0);
4866 default: return Register();
4867 }
4868}
4869
4870// FastEmit functions for X86ISD::CVTSI2P.
4871
4872Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
4873 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4874 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4875 }
4876 return Register();
4877}
4878
4879Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
4880 if ((Subtarget->hasVLX())) {
4881 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
4882 }
4883 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4884 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4885 }
4886 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4887 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4888 }
4889 return Register();
4890}
4891
4892Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
4893switch (RetVT.SimpleTy) {
4894 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
4895 case MVT::v2f64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
4896 default: return Register();
4897}
4898}
4899
4900Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
4901 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4902 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4903 }
4904 return Register();
4905}
4906
4907Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
4908 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4909 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4910 }
4911 return Register();
4912}
4913
4914Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
4915switch (RetVT.SimpleTy) {
4916 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
4917 case MVT::v4f32: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
4918 default: return Register();
4919}
4920}
4921
4922Register fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
4923 if (RetVT.SimpleTy != MVT::v8f16)
4924 return Register();
4925 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4926 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
4927 }
4928 return Register();
4929}
4930
4931Register fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, Register Op0) {
4932 switch (VT.SimpleTy) {
4933 case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
4934 case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
4935 case MVT::v4i64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
4936 default: return Register();
4937 }
4938}
4939
4940// FastEmit functions for X86ISD::CVTTP2IBS.
4941
4942Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
4943 if (RetVT.SimpleTy != MVT::v8i16)
4944 return Register();
4945 if ((Subtarget->hasAVX10_2())) {
4946 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4947 }
4948 return Register();
4949}
4950
4951Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4952 if (RetVT.SimpleTy != MVT::v16i16)
4953 return Register();
4954 if ((Subtarget->hasAVX10_2())) {
4955 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4956 }
4957 return Register();
4958}
4959
4960Register fastEmit_X86ISD_CVTTP2IBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4961 if (RetVT.SimpleTy != MVT::v32i16)
4962 return Register();
4963 if ((Subtarget->hasAVX10_2())) {
4964 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZrr, RC: &X86::VR512RegClass, Op0);
4965 }
4966 return Register();
4967}
4968
4969Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
4970 if (RetVT.SimpleTy != MVT::v8i16)
4971 return Register();
4972 if ((Subtarget->hasAVX10_2())) {
4973 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4974 }
4975 return Register();
4976}
4977
4978Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
4979 if (RetVT.SimpleTy != MVT::v16i16)
4980 return Register();
4981 if ((Subtarget->hasAVX10_2())) {
4982 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4983 }
4984 return Register();
4985}
4986
4987Register fastEmit_X86ISD_CVTTP2IBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4988 if (RetVT.SimpleTy != MVT::v32i16)
4989 return Register();
4990 if ((Subtarget->hasAVX10_2())) {
4991 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZrr, RC: &X86::VR512RegClass, Op0);
4992 }
4993 return Register();
4994}
4995
4996Register fastEmit_X86ISD_CVTTP2IBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4997 if (RetVT.SimpleTy != MVT::v4i32)
4998 return Register();
4999 if ((Subtarget->hasAVX10_2())) {
5000 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5001 }
5002 return Register();
5003}
5004
5005Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5006 if (RetVT.SimpleTy != MVT::v8i32)
5007 return Register();
5008 if ((Subtarget->hasAVX10_2())) {
5009 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5010 }
5011 return Register();
5012}
5013
5014Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5015 if (RetVT.SimpleTy != MVT::v16i32)
5016 return Register();
5017 if ((Subtarget->hasAVX10_2())) {
5018 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZrr, RC: &X86::VR512RegClass, Op0);
5019 }
5020 return Register();
5021}
5022
5023Register fastEmit_X86ISD_CVTTP2IBS_r(MVT VT, MVT RetVT, Register Op0) {
5024 switch (VT.SimpleTy) {
5025 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8f16_r(RetVT, Op0);
5026 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16f16_r(RetVT, Op0);
5027 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v32f16_r(RetVT, Op0);
5028 case MVT::v8bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8bf16_r(RetVT, Op0);
5029 case MVT::v16bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16bf16_r(RetVT, Op0);
5030 case MVT::v32bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v32bf16_r(RetVT, Op0);
5031 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v4f32_r(RetVT, Op0);
5032 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8f32_r(RetVT, Op0);
5033 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16f32_r(RetVT, Op0);
5034 default: return Register();
5035 }
5036}
5037
5038// FastEmit functions for X86ISD::CVTTP2IBS_SAE.
5039
5040Register fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5041 if (RetVT.SimpleTy != MVT::v32i16)
5042 return Register();
5043 if ((Subtarget->hasAVX10_2())) {
5044 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZrrb, RC: &X86::VR512RegClass, Op0);
5045 }
5046 return Register();
5047}
5048
5049Register fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5050 if (RetVT.SimpleTy != MVT::v16i32)
5051 return Register();
5052 if ((Subtarget->hasAVX10_2())) {
5053 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZrrb, RC: &X86::VR512RegClass, Op0);
5054 }
5055 return Register();
5056}
5057
5058Register fastEmit_X86ISD_CVTTP2IBS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5059 switch (VT.SimpleTy) {
5060 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v32f16_r(RetVT, Op0);
5061 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v16f32_r(RetVT, Op0);
5062 default: return Register();
5063 }
5064}
5065
5066// FastEmit functions for X86ISD::CVTTP2IUBS.
5067
5068Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
5069 if (RetVT.SimpleTy != MVT::v8i16)
5070 return Register();
5071 if ((Subtarget->hasAVX10_2())) {
5072 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5073 }
5074 return Register();
5075}
5076
5077Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
5078 if (RetVT.SimpleTy != MVT::v16i16)
5079 return Register();
5080 if ((Subtarget->hasAVX10_2())) {
5081 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5082 }
5083 return Register();
5084}
5085
5086Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
5087 if (RetVT.SimpleTy != MVT::v32i16)
5088 return Register();
5089 if ((Subtarget->hasAVX10_2())) {
5090 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZrr, RC: &X86::VR512RegClass, Op0);
5091 }
5092 return Register();
5093}
5094
5095Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
5096 if (RetVT.SimpleTy != MVT::v8i16)
5097 return Register();
5098 if ((Subtarget->hasAVX10_2())) {
5099 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5100 }
5101 return Register();
5102}
5103
5104Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
5105 if (RetVT.SimpleTy != MVT::v16i16)
5106 return Register();
5107 if ((Subtarget->hasAVX10_2())) {
5108 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5109 }
5110 return Register();
5111}
5112
5113Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
5114 if (RetVT.SimpleTy != MVT::v32i16)
5115 return Register();
5116 if ((Subtarget->hasAVX10_2())) {
5117 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZrr, RC: &X86::VR512RegClass, Op0);
5118 }
5119 return Register();
5120}
5121
5122Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5123 if (RetVT.SimpleTy != MVT::v4i32)
5124 return Register();
5125 if ((Subtarget->hasAVX10_2())) {
5126 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5127 }
5128 return Register();
5129}
5130
5131Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5132 if (RetVT.SimpleTy != MVT::v8i32)
5133 return Register();
5134 if ((Subtarget->hasAVX10_2())) {
5135 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5136 }
5137 return Register();
5138}
5139
5140Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5141 if (RetVT.SimpleTy != MVT::v16i32)
5142 return Register();
5143 if ((Subtarget->hasAVX10_2())) {
5144 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZrr, RC: &X86::VR512RegClass, Op0);
5145 }
5146 return Register();
5147}
5148
5149Register fastEmit_X86ISD_CVTTP2IUBS_r(MVT VT, MVT RetVT, Register Op0) {
5150 switch (VT.SimpleTy) {
5151 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f16_r(RetVT, Op0);
5152 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f16_r(RetVT, Op0);
5153 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v32f16_r(RetVT, Op0);
5154 case MVT::v8bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8bf16_r(RetVT, Op0);
5155 case MVT::v16bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16bf16_r(RetVT, Op0);
5156 case MVT::v32bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v32bf16_r(RetVT, Op0);
5157 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v4f32_r(RetVT, Op0);
5158 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f32_r(RetVT, Op0);
5159 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f32_r(RetVT, Op0);
5160 default: return Register();
5161 }
5162}
5163
5164// FastEmit functions for X86ISD::CVTTP2IUBS_SAE.
5165
5166Register fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5167 if (RetVT.SimpleTy != MVT::v32i16)
5168 return Register();
5169 if ((Subtarget->hasAVX10_2())) {
5170 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZrrb, RC: &X86::VR512RegClass, Op0);
5171 }
5172 return Register();
5173}
5174
5175Register fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5176 if (RetVT.SimpleTy != MVT::v16i32)
5177 return Register();
5178 if ((Subtarget->hasAVX10_2())) {
5179 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZrrb, RC: &X86::VR512RegClass, Op0);
5180 }
5181 return Register();
5182}
5183
5184Register fastEmit_X86ISD_CVTTP2IUBS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5185 switch (VT.SimpleTy) {
5186 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v32f16_r(RetVT, Op0);
5187 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v16f32_r(RetVT, Op0);
5188 default: return Register();
5189 }
5190}
5191
5192// FastEmit functions for X86ISD::CVTTP2SI.
5193
5194Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
5195 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5196 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
5197 }
5198 return Register();
5199}
5200
5201Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
5202 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5203 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5204 }
5205 return Register();
5206}
5207
5208Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
5209 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5210 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
5211 }
5212 return Register();
5213}
5214
5215Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
5216 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5217 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5218 }
5219 return Register();
5220}
5221
5222Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
5223 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5224 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5225 }
5226 return Register();
5227}
5228
5229Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
5230 if ((Subtarget->hasFP16())) {
5231 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
5232 }
5233 return Register();
5234}
5235
5236Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
5237switch (RetVT.SimpleTy) {
5238 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
5239 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
5240 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
5241 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
5242 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
5243 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
5244 default: return Register();
5245}
5246}
5247
5248Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
5249 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5250 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
5251 }
5252 return Register();
5253}
5254
5255Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
5256 if ((Subtarget->hasFP16())) {
5257 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
5258 }
5259 return Register();
5260}
5261
5262Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
5263switch (RetVT.SimpleTy) {
5264 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
5265 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
5266 default: return Register();
5267}
5268}
5269
5270Register fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
5271 if (RetVT.SimpleTy != MVT::v32i16)
5272 return Register();
5273 if ((Subtarget->hasFP16())) {
5274 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
5275 }
5276 return Register();
5277}
5278
5279Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5280 if ((Subtarget->hasVLX())) {
5281 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5282 }
5283 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5284 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
5285 }
5286 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5287 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
5288 }
5289 return Register();
5290}
5291
5292Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5293 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5294 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5295 }
5296 return Register();
5297}
5298
5299Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5300 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5301 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5302 }
5303 return Register();
5304}
5305
5306Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
5307switch (RetVT.SimpleTy) {
5308 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
5309 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
5310 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
5311 default: return Register();
5312}
5313}
5314
5315Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5316 if ((Subtarget->hasVLX())) {
5317 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
5318 }
5319 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5320 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
5321 }
5322 return Register();
5323}
5324
5325Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5326 if ((Subtarget->hasDQI())) {
5327 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
5328 }
5329 return Register();
5330}
5331
5332Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
5333switch (RetVT.SimpleTy) {
5334 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
5335 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
5336 default: return Register();
5337}
5338}
5339
5340Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
5341 if (RetVT.SimpleTy != MVT::v16i32)
5342 return Register();
5343 if ((Subtarget->hasAVX512())) {
5344 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
5345 }
5346 return Register();
5347}
5348
5349Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5350 if ((Subtarget->hasVLX())) {
5351 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5352 }
5353 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5354 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
5355 }
5356 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
5358 }
5359 return Register();
5360}
5361
5362Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5363 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5364 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5365 }
5366 return Register();
5367}
5368
5369Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
5370switch (RetVT.SimpleTy) {
5371 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
5372 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
5373 default: return Register();
5374}
5375}
5376
5377Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5378 if ((Subtarget->hasVLX())) {
5379 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
5380 }
5381 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5382 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
5383 }
5384 return Register();
5385}
5386
5387Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5388 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5389 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5390 }
5391 return Register();
5392}
5393
5394Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
5395switch (RetVT.SimpleTy) {
5396 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
5397 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
5398 default: return Register();
5399}
5400}
5401
5402Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5403 if ((Subtarget->hasAVX512())) {
5404 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
5405 }
5406 return Register();
5407}
5408
5409Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5410 if ((Subtarget->hasDQI())) {
5411 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
5412 }
5413 return Register();
5414}
5415
5416Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
5417switch (RetVT.SimpleTy) {
5418 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
5419 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
5420 default: return Register();
5421}
5422}
5423
5424Register fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
5425 switch (VT.SimpleTy) {
5426 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
5427 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
5428 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
5429 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
5430 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
5431 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
5432 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
5433 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
5434 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
5435 default: return Register();
5436 }
5437}
5438
5439// FastEmit functions for X86ISD::CVTTP2SIS.
5440
5441Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5442 if ((Subtarget->hasAVX10_2())) {
5443 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5444 }
5445 return Register();
5446}
5447
5448Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5449 if ((Subtarget->hasAVX10_2())) {
5450 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5451 }
5452 return Register();
5453}
5454
5455Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5456 if ((Subtarget->hasAVX10_2())) {
5457 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5458 }
5459 return Register();
5460}
5461
5462Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5463switch (RetVT.SimpleTy) {
5464 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i32_r(Op0);
5465 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v2i64_r(Op0);
5466 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i64_r(Op0);
5467 default: return Register();
5468}
5469}
5470
5471Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5472 if ((Subtarget->hasAVX10_2())) {
5473 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5474 }
5475 return Register();
5476}
5477
5478Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5479 if ((Subtarget->hasAVX10_2())) {
5480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZrr, RC: &X86::VR512RegClass, Op0);
5481 }
5482 return Register();
5483}
5484
5485Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5486switch (RetVT.SimpleTy) {
5487 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i32_r(Op0);
5488 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i64_r(Op0);
5489 default: return Register();
5490}
5491}
5492
5493Register fastEmit_X86ISD_CVTTP2SIS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5494 if (RetVT.SimpleTy != MVT::v16i32)
5495 return Register();
5496 if ((Subtarget->hasAVX10_2())) {
5497 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZrr, RC: &X86::VR512RegClass, Op0);
5498 }
5499 return Register();
5500}
5501
5502Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5503 if ((Subtarget->hasAVX10_2())) {
5504 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5505 }
5506 return Register();
5507}
5508
5509Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5510 if ((Subtarget->hasAVX10_2())) {
5511 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5512 }
5513 return Register();
5514}
5515
5516Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
5517switch (RetVT.SimpleTy) {
5518 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v4i32_r(Op0);
5519 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v2i64_r(Op0);
5520 default: return Register();
5521}
5522}
5523
5524Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5525 if ((Subtarget->hasAVX10_2())) {
5526 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ256rr, RC: &X86::VR128XRegClass, Op0);
5527 }
5528 return Register();
5529}
5530
5531Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5532 if ((Subtarget->hasAVX10_2())) {
5533 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5534 }
5535 return Register();
5536}
5537
5538Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_r(MVT RetVT, Register Op0) {
5539switch (RetVT.SimpleTy) {
5540 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i32_r(Op0);
5541 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i64_r(Op0);
5542 default: return Register();
5543}
5544}
5545
5546Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5547 if ((Subtarget->hasAVX10_2())) {
5548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZrr, RC: &X86::VR256XRegClass, Op0);
5549 }
5550 return Register();
5551}
5552
5553Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5554 if ((Subtarget->hasAVX10_2())) {
5555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZrr, RC: &X86::VR512RegClass, Op0);
5556 }
5557 return Register();
5558}
5559
5560Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_r(MVT RetVT, Register Op0) {
5561switch (RetVT.SimpleTy) {
5562 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i32_r(Op0);
5563 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i64_r(Op0);
5564 default: return Register();
5565}
5566}
5567
5568Register fastEmit_X86ISD_CVTTP2SIS_r(MVT VT, MVT RetVT, Register Op0) {
5569 switch (VT.SimpleTy) {
5570 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_r(RetVT, Op0);
5571 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_r(RetVT, Op0);
5572 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v16f32_r(RetVT, Op0);
5573 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_r(RetVT, Op0);
5574 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_r(RetVT, Op0);
5575 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_r(RetVT, Op0);
5576 default: return Register();
5577 }
5578}
5579
5580// FastEmit functions for X86ISD::CVTTP2SIS_SAE.
5581
5582Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
5583 if (RetVT.SimpleTy != MVT::v4i64)
5584 return Register();
5585 if ((Subtarget->hasAVX10_2())) {
5586 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
5587 }
5588 return Register();
5589}
5590
5591Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
5592 if (RetVT.SimpleTy != MVT::v8i64)
5593 return Register();
5594 if ((Subtarget->hasAVX10_2())) {
5595 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZrrb, RC: &X86::VR512RegClass, Op0);
5596 }
5597 return Register();
5598}
5599
5600Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5601 if (RetVT.SimpleTy != MVT::v16i32)
5602 return Register();
5603 if ((Subtarget->hasAVX10_2())) {
5604 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZrrb, RC: &X86::VR512RegClass, Op0);
5605 }
5606 return Register();
5607}
5608
5609Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5610 if ((Subtarget->hasAVX10_2())) {
5611 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ256rrb, RC: &X86::VR128XRegClass, Op0);
5612 }
5613 return Register();
5614}
5615
5616Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5617 if ((Subtarget->hasAVX10_2())) {
5618 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
5619 }
5620 return Register();
5621}
5622
5623Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_r(MVT RetVT, Register Op0) {
5624switch (RetVT.SimpleTy) {
5625 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i32_r(Op0);
5626 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i64_r(Op0);
5627 default: return Register();
5628}
5629}
5630
5631Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5632 if ((Subtarget->hasAVX10_2())) {
5633 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZrrb, RC: &X86::VR256XRegClass, Op0);
5634 }
5635 return Register();
5636}
5637
5638Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5639 if ((Subtarget->hasAVX10_2())) {
5640 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZrrb, RC: &X86::VR512RegClass, Op0);
5641 }
5642 return Register();
5643}
5644
5645Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
5646switch (RetVT.SimpleTy) {
5647 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5648 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5649 default: return Register();
5650}
5651}
5652
5653Register fastEmit_X86ISD_CVTTP2SIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5654 switch (VT.SimpleTy) {
5655 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f32_r(RetVT, Op0);
5656 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f32_r(RetVT, Op0);
5657 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v16f32_r(RetVT, Op0);
5658 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_r(RetVT, Op0);
5659 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_r(RetVT, Op0);
5660 default: return Register();
5661 }
5662}
5663
5664// FastEmit functions for X86ISD::CVTTP2SI_SAE.
5665
5666Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
5667 if (RetVT.SimpleTy != MVT::v8i64)
5668 return Register();
5669 if ((Subtarget->hasFP16())) {
5670 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrrb, RC: &X86::VR512RegClass, Op0);
5671 }
5672 return Register();
5673}
5674
5675Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
5676 if (RetVT.SimpleTy != MVT::v16i32)
5677 return Register();
5678 if ((Subtarget->hasFP16())) {
5679 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrrb, RC: &X86::VR512RegClass, Op0);
5680 }
5681 return Register();
5682}
5683
5684Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5685 if (RetVT.SimpleTy != MVT::v32i16)
5686 return Register();
5687 if ((Subtarget->hasFP16())) {
5688 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrrb, RC: &X86::VR512RegClass, Op0);
5689 }
5690 return Register();
5691}
5692
5693Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
5694 if (RetVT.SimpleTy != MVT::v8i64)
5695 return Register();
5696 if ((Subtarget->hasDQI())) {
5697 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrrb, RC: &X86::VR512RegClass, Op0);
5698 }
5699 return Register();
5700}
5701
5702Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5703 if (RetVT.SimpleTy != MVT::v16i32)
5704 return Register();
5705 if ((Subtarget->hasAVX512())) {
5706 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrrb, RC: &X86::VR512RegClass, Op0);
5707 }
5708 return Register();
5709}
5710
5711Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5712 if ((Subtarget->hasAVX512())) {
5713 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrrb, RC: &X86::VR256XRegClass, Op0);
5714 }
5715 return Register();
5716}
5717
5718Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5719 if ((Subtarget->hasDQI())) {
5720 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrrb, RC: &X86::VR512RegClass, Op0);
5721 }
5722 return Register();
5723}
5724
5725Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
5726switch (RetVT.SimpleTy) {
5727 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5728 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5729 default: return Register();
5730}
5731}
5732
5733Register fastEmit_X86ISD_CVTTP2SI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5734 switch (VT.SimpleTy) {
5735 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(RetVT, Op0);
5736 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(RetVT, Op0);
5737 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(RetVT, Op0);
5738 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(RetVT, Op0);
5739 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(RetVT, Op0);
5740 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(RetVT, Op0);
5741 default: return Register();
5742 }
5743}
5744
5745// FastEmit functions for X86ISD::CVTTP2UI.
5746
5747Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
5748 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5749 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
5750 }
5751 return Register();
5752}
5753
5754Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
5755 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5756 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5757 }
5758 return Register();
5759}
5760
5761Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
5762 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5763 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5764 }
5765 return Register();
5766}
5767
5768Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
5769 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5770 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5771 }
5772 return Register();
5773}
5774
5775Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
5776 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5777 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5778 }
5779 return Register();
5780}
5781
5782Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
5783 if ((Subtarget->hasFP16())) {
5784 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
5785 }
5786 return Register();
5787}
5788
5789Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
5790switch (RetVT.SimpleTy) {
5791 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
5792 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
5793 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
5794 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
5795 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
5796 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
5797 default: return Register();
5798}
5799}
5800
5801Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
5802 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5803 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
5804 }
5805 return Register();
5806}
5807
5808Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
5809 if ((Subtarget->hasFP16())) {
5810 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
5811 }
5812 return Register();
5813}
5814
5815Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
5816switch (RetVT.SimpleTy) {
5817 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
5818 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
5819 default: return Register();
5820}
5821}
5822
5823Register fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
5824 if (RetVT.SimpleTy != MVT::v32i16)
5825 return Register();
5826 if ((Subtarget->hasFP16())) {
5827 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
5828 }
5829 return Register();
5830}
5831
5832Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5833 if ((Subtarget->hasVLX())) {
5834 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5835 }
5836 return Register();
5837}
5838
5839Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5840 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5841 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5842 }
5843 return Register();
5844}
5845
5846Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5847 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5848 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5849 }
5850 return Register();
5851}
5852
5853Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
5854switch (RetVT.SimpleTy) {
5855 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
5856 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
5857 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
5858 default: return Register();
5859}
5860}
5861
5862Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5863 if ((Subtarget->hasVLX())) {
5864 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5865 }
5866 return Register();
5867}
5868
5869Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5870 if ((Subtarget->hasDQI())) {
5871 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
5872 }
5873 return Register();
5874}
5875
5876Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
5877switch (RetVT.SimpleTy) {
5878 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
5879 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
5880 default: return Register();
5881}
5882}
5883
5884Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
5885 if (RetVT.SimpleTy != MVT::v16i32)
5886 return Register();
5887 if ((Subtarget->hasAVX512())) {
5888 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
5889 }
5890 return Register();
5891}
5892
5893Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5894 if ((Subtarget->hasVLX())) {
5895 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5896 }
5897 return Register();
5898}
5899
5900Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5901 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5902 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5903 }
5904 return Register();
5905}
5906
5907Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
5908switch (RetVT.SimpleTy) {
5909 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
5910 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
5911 default: return Register();
5912}
5913}
5914
5915Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5916 if ((Subtarget->hasVLX())) {
5917 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
5918 }
5919 return Register();
5920}
5921
5922Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5923 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5924 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5925 }
5926 return Register();
5927}
5928
5929Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
5930switch (RetVT.SimpleTy) {
5931 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
5932 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
5933 default: return Register();
5934}
5935}
5936
5937Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5938 if ((Subtarget->hasAVX512())) {
5939 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
5940 }
5941 return Register();
5942}
5943
5944Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5945 if ((Subtarget->hasDQI())) {
5946 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
5947 }
5948 return Register();
5949}
5950
5951Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
5952switch (RetVT.SimpleTy) {
5953 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
5954 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
5955 default: return Register();
5956}
5957}
5958
5959Register fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
5960 switch (VT.SimpleTy) {
5961 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
5962 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
5963 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
5964 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
5965 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
5966 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
5967 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
5968 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
5969 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
5970 default: return Register();
5971 }
5972}
5973
5974// FastEmit functions for X86ISD::CVTTP2UIS.
5975
5976Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5977 if ((Subtarget->hasAVX10_2())) {
5978 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5979 }
5980 return Register();
5981}
5982
5983Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5984 if ((Subtarget->hasAVX10_2())) {
5985 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5986 }
5987 return Register();
5988}
5989
5990Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5991 if ((Subtarget->hasAVX10_2())) {
5992 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5993 }
5994 return Register();
5995}
5996
5997Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5998switch (RetVT.SimpleTy) {
5999 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i32_r(Op0);
6000 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v2i64_r(Op0);
6001 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i64_r(Op0);
6002 default: return Register();
6003}
6004}
6005
6006Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i32_r(Register Op0) {
6007 if ((Subtarget->hasAVX10_2())) {
6008 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZ256rr, RC: &X86::VR256XRegClass, Op0);
6009 }
6010 return Register();
6011}
6012
6013Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i64_r(Register Op0) {
6014 if ((Subtarget->hasAVX10_2())) {
6015 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZrr, RC: &X86::VR512RegClass, Op0);
6016 }
6017 return Register();
6018}
6019
6020Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_r(MVT RetVT, Register Op0) {
6021switch (RetVT.SimpleTy) {
6022 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i32_r(Op0);
6023 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i64_r(Op0);
6024 default: return Register();
6025}
6026}
6027
6028Register fastEmit_X86ISD_CVTTP2UIS_MVT_v16f32_r(MVT RetVT, Register Op0) {
6029 if (RetVT.SimpleTy != MVT::v16i32)
6030 return Register();
6031 if ((Subtarget->hasAVX10_2())) {
6032 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZrr, RC: &X86::VR512RegClass, Op0);
6033 }
6034 return Register();
6035}
6036
6037Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v4i32_r(Register Op0) {
6038 if ((Subtarget->hasAVX10_2())) {
6039 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6040 }
6041 return Register();
6042}
6043
6044Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v2i64_r(Register Op0) {
6045 if ((Subtarget->hasAVX10_2())) {
6046 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6047 }
6048 return Register();
6049}
6050
6051Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6052switch (RetVT.SimpleTy) {
6053 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v4i32_r(Op0);
6054 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v2i64_r(Op0);
6055 default: return Register();
6056}
6057}
6058
6059Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i32_r(Register Op0) {
6060 if ((Subtarget->hasAVX10_2())) {
6061 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ256rr, RC: &X86::VR128XRegClass, Op0);
6062 }
6063 return Register();
6064}
6065
6066Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i64_r(Register Op0) {
6067 if ((Subtarget->hasAVX10_2())) {
6068 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ256rr, RC: &X86::VR256XRegClass, Op0);
6069 }
6070 return Register();
6071}
6072
6073Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_r(MVT RetVT, Register Op0) {
6074switch (RetVT.SimpleTy) {
6075 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i32_r(Op0);
6076 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i64_r(Op0);
6077 default: return Register();
6078}
6079}
6080
6081Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6082 if ((Subtarget->hasAVX10_2())) {
6083 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZrr, RC: &X86::VR256XRegClass, Op0);
6084 }
6085 return Register();
6086}
6087
6088Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6089 if ((Subtarget->hasAVX10_2())) {
6090 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZrr, RC: &X86::VR512RegClass, Op0);
6091 }
6092 return Register();
6093}
6094
6095Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_r(MVT RetVT, Register Op0) {
6096switch (RetVT.SimpleTy) {
6097 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i32_r(Op0);
6098 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i64_r(Op0);
6099 default: return Register();
6100}
6101}
6102
6103Register fastEmit_X86ISD_CVTTP2UIS_r(MVT VT, MVT RetVT, Register Op0) {
6104 switch (VT.SimpleTy) {
6105 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_r(RetVT, Op0);
6106 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_r(RetVT, Op0);
6107 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v16f32_r(RetVT, Op0);
6108 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_r(RetVT, Op0);
6109 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_r(RetVT, Op0);
6110 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_r(RetVT, Op0);
6111 default: return Register();
6112 }
6113}
6114
6115// FastEmit functions for X86ISD::CVTTP2UIS_SAE.
6116
6117Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6118 if (RetVT.SimpleTy != MVT::v4i64)
6119 return Register();
6120 if ((Subtarget->hasAVX10_2())) {
6121 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
6122 }
6123 return Register();
6124}
6125
6126Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
6127 if (RetVT.SimpleTy != MVT::v8i64)
6128 return Register();
6129 if ((Subtarget->hasAVX10_2())) {
6130 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZrrb, RC: &X86::VR512RegClass, Op0);
6131 }
6132 return Register();
6133}
6134
6135Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6136 if (RetVT.SimpleTy != MVT::v16i32)
6137 return Register();
6138 if ((Subtarget->hasAVX10_2())) {
6139 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZrrb, RC: &X86::VR512RegClass, Op0);
6140 }
6141 return Register();
6142}
6143
6144Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i32_r(Register Op0) {
6145 if ((Subtarget->hasAVX10_2())) {
6146 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ256rrb, RC: &X86::VR128XRegClass, Op0);
6147 }
6148 return Register();
6149}
6150
6151Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i64_r(Register Op0) {
6152 if ((Subtarget->hasAVX10_2())) {
6153 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
6154 }
6155 return Register();
6156}
6157
6158Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_r(MVT RetVT, Register Op0) {
6159switch (RetVT.SimpleTy) {
6160 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i32_r(Op0);
6161 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i64_r(Op0);
6162 default: return Register();
6163}
6164}
6165
6166Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6167 if ((Subtarget->hasAVX10_2())) {
6168 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZrrb, RC: &X86::VR256XRegClass, Op0);
6169 }
6170 return Register();
6171}
6172
6173Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6174 if ((Subtarget->hasAVX10_2())) {
6175 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZrrb, RC: &X86::VR512RegClass, Op0);
6176 }
6177 return Register();
6178}
6179
6180Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6181switch (RetVT.SimpleTy) {
6182 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
6183 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
6184 default: return Register();
6185}
6186}
6187
6188Register fastEmit_X86ISD_CVTTP2UIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6189 switch (VT.SimpleTy) {
6190 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f32_r(RetVT, Op0);
6191 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f32_r(RetVT, Op0);
6192 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v16f32_r(RetVT, Op0);
6193 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_r(RetVT, Op0);
6194 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_r(RetVT, Op0);
6195 default: return Register();
6196 }
6197}
6198
6199// FastEmit functions for X86ISD::CVTTP2UI_SAE.
6200
6201Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6202 if (RetVT.SimpleTy != MVT::v8i64)
6203 return Register();
6204 if ((Subtarget->hasFP16())) {
6205 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6206 }
6207 return Register();
6208}
6209
6210Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
6211 if (RetVT.SimpleTy != MVT::v16i32)
6212 return Register();
6213 if ((Subtarget->hasFP16())) {
6214 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrrb, RC: &X86::VR512RegClass, Op0);
6215 }
6216 return Register();
6217}
6218
6219Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
6220 if (RetVT.SimpleTy != MVT::v32i16)
6221 return Register();
6222 if ((Subtarget->hasFP16())) {
6223 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrrb, RC: &X86::VR512RegClass, Op0);
6224 }
6225 return Register();
6226}
6227
6228Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
6229 if (RetVT.SimpleTy != MVT::v8i64)
6230 return Register();
6231 if ((Subtarget->hasDQI())) {
6232 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6233 }
6234 return Register();
6235}
6236
6237Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6238 if (RetVT.SimpleTy != MVT::v16i32)
6239 return Register();
6240 if ((Subtarget->hasAVX512())) {
6241 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrrb, RC: &X86::VR512RegClass, Op0);
6242 }
6243 return Register();
6244}
6245
6246Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6247 if ((Subtarget->hasAVX512())) {
6248 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrrb, RC: &X86::VR256XRegClass, Op0);
6249 }
6250 return Register();
6251}
6252
6253Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6254 if ((Subtarget->hasDQI())) {
6255 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6256 }
6257 return Register();
6258}
6259
6260Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6261switch (RetVT.SimpleTy) {
6262 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
6263 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
6264 default: return Register();
6265}
6266}
6267
6268Register fastEmit_X86ISD_CVTTP2UI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6269 switch (VT.SimpleTy) {
6270 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(RetVT, Op0);
6271 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(RetVT, Op0);
6272 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(RetVT, Op0);
6273 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(RetVT, Op0);
6274 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(RetVT, Op0);
6275 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(RetVT, Op0);
6276 default: return Register();
6277 }
6278}
6279
6280// FastEmit functions for X86ISD::CVTTS2SI.
6281
6282Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Register Op0) {
6283 if ((Subtarget->hasFP16())) {
6284 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6285 }
6286 return Register();
6287}
6288
6289Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Register Op0) {
6290 if ((Subtarget->hasFP16())) {
6291 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6292 }
6293 return Register();
6294}
6295
6296Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
6297switch (RetVT.SimpleTy) {
6298 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Op0);
6299 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Op0);
6300 default: return Register();
6301}
6302}
6303
6304Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Register Op0) {
6305 if ((Subtarget->hasAVX512())) {
6306 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6307 }
6308 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6309 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6310 }
6311 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6312 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6313 }
6314 return Register();
6315}
6316
6317Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Register Op0) {
6318 if ((Subtarget->hasAVX512())) {
6319 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6320 }
6321 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6322 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6323 }
6324 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6325 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6326 }
6327 return Register();
6328}
6329
6330Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
6331switch (RetVT.SimpleTy) {
6332 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0);
6333 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0);
6334 default: return Register();
6335}
6336}
6337
6338Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Register Op0) {
6339 if ((Subtarget->hasAVX512())) {
6340 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6341 }
6342 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6343 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6344 }
6345 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6346 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6347 }
6348 return Register();
6349}
6350
6351Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Register Op0) {
6352 if ((Subtarget->hasAVX512())) {
6353 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6354 }
6355 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6356 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6357 }
6358 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6359 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6360 }
6361 return Register();
6362}
6363
6364Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
6365switch (RetVT.SimpleTy) {
6366 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0);
6367 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0);
6368 default: return Register();
6369}
6370}
6371
6372Register fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, Register Op0) {
6373 switch (VT.SimpleTy) {
6374 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(RetVT, Op0);
6375 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0);
6376 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0);
6377 default: return Register();
6378 }
6379}
6380
6381// FastEmit functions for X86ISD::CVTTS2SIS.
6382
6383Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i32_r(Register Op0) {
6384 if ((Subtarget->hasAVX10_2())) {
6385 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SISrr_Int, RC: &X86::GR32RegClass, Op0);
6386 }
6387 return Register();
6388}
6389
6390Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i64_r(Register Op0) {
6391 if ((Subtarget->hasAVX10_2())) {
6392 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6393 }
6394 return Register();
6395}
6396
6397Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6398switch (RetVT.SimpleTy) {
6399 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i32_r(Op0);
6400 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i64_r(Op0);
6401 default: return Register();
6402}
6403}
6404
6405Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i32_r(Register Op0) {
6406 if ((Subtarget->hasAVX10_2())) {
6407 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SISrr_Int, RC: &X86::GR32RegClass, Op0);
6408 }
6409 return Register();
6410}
6411
6412Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i64_r(Register Op0) {
6413 if ((Subtarget->hasAVX10_2())) {
6414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6415 }
6416 return Register();
6417}
6418
6419Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6420switch (RetVT.SimpleTy) {
6421 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i32_r(Op0);
6422 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i64_r(Op0);
6423 default: return Register();
6424}
6425}
6426
6427Register fastEmit_X86ISD_CVTTS2SIS_r(MVT VT, MVT RetVT, Register Op0) {
6428 switch (VT.SimpleTy) {
6429 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_r(RetVT, Op0);
6430 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_r(RetVT, Op0);
6431 default: return Register();
6432 }
6433}
6434
6435// FastEmit functions for X86ISD::CVTTS2SIS_SAE.
6436
6437Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6438 if ((Subtarget->hasAVX10_2())) {
6439 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SISrrb_Int, RC: &X86::GR32RegClass, Op0);
6440 }
6441 return Register();
6442}
6443
6444Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6445 if ((Subtarget->hasAVX10_2())) {
6446 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6447 }
6448 return Register();
6449}
6450
6451Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6452switch (RetVT.SimpleTy) {
6453 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i32_r(Op0);
6454 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i64_r(Op0);
6455 default: return Register();
6456}
6457}
6458
6459Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6460 if ((Subtarget->hasAVX10_2())) {
6461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SISrrb_Int, RC: &X86::GR32RegClass, Op0);
6462 }
6463 return Register();
6464}
6465
6466Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6467 if ((Subtarget->hasAVX10_2())) {
6468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6469 }
6470 return Register();
6471}
6472
6473Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6474switch (RetVT.SimpleTy) {
6475 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i32_r(Op0);
6476 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i64_r(Op0);
6477 default: return Register();
6478}
6479}
6480
6481Register fastEmit_X86ISD_CVTTS2SIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6482 switch (VT.SimpleTy) {
6483 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_r(RetVT, Op0);
6484 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_r(RetVT, Op0);
6485 default: return Register();
6486 }
6487}
6488
6489// FastEmit functions for X86ISD::CVTTS2SI_SAE.
6490
6491Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Register Op0) {
6492 if ((Subtarget->hasFP16())) {
6493 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6494 }
6495 return Register();
6496}
6497
6498Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Register Op0) {
6499 if ((Subtarget->hasFP16())) {
6500 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6501 }
6502 return Register();
6503}
6504
6505Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6506switch (RetVT.SimpleTy) {
6507 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Op0);
6508 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Op0);
6509 default: return Register();
6510}
6511}
6512
6513Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6514 if ((Subtarget->hasAVX512())) {
6515 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6516 }
6517 return Register();
6518}
6519
6520Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6521 if ((Subtarget->hasAVX512())) {
6522 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6523 }
6524 return Register();
6525}
6526
6527Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6528switch (RetVT.SimpleTy) {
6529 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Op0);
6530 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Op0);
6531 default: return Register();
6532}
6533}
6534
6535Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6536 if ((Subtarget->hasAVX512())) {
6537 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6538 }
6539 return Register();
6540}
6541
6542Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6543 if ((Subtarget->hasAVX512())) {
6544 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6545 }
6546 return Register();
6547}
6548
6549Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6550switch (RetVT.SimpleTy) {
6551 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Op0);
6552 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Op0);
6553 default: return Register();
6554}
6555}
6556
6557Register fastEmit_X86ISD_CVTTS2SI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6558 switch (VT.SimpleTy) {
6559 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(RetVT, Op0);
6560 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(RetVT, Op0);
6561 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(RetVT, Op0);
6562 default: return Register();
6563 }
6564}
6565
6566// FastEmit functions for X86ISD::CVTTS2UI.
6567
6568Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Register Op0) {
6569 if ((Subtarget->hasFP16())) {
6570 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6571 }
6572 return Register();
6573}
6574
6575Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Register Op0) {
6576 if ((Subtarget->hasFP16())) {
6577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6578 }
6579 return Register();
6580}
6581
6582Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
6583switch (RetVT.SimpleTy) {
6584 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Op0);
6585 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Op0);
6586 default: return Register();
6587}
6588}
6589
6590Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Register Op0) {
6591 if ((Subtarget->hasAVX512())) {
6592 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6593 }
6594 return Register();
6595}
6596
6597Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Register Op0) {
6598 if ((Subtarget->hasAVX512())) {
6599 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6600 }
6601 return Register();
6602}
6603
6604Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
6605switch (RetVT.SimpleTy) {
6606 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0);
6607 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0);
6608 default: return Register();
6609}
6610}
6611
6612Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Register Op0) {
6613 if ((Subtarget->hasAVX512())) {
6614 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6615 }
6616 return Register();
6617}
6618
6619Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Register Op0) {
6620 if ((Subtarget->hasAVX512())) {
6621 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6622 }
6623 return Register();
6624}
6625
6626Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
6627switch (RetVT.SimpleTy) {
6628 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0);
6629 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0);
6630 default: return Register();
6631}
6632}
6633
6634Register fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, Register Op0) {
6635 switch (VT.SimpleTy) {
6636 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(RetVT, Op0);
6637 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0);
6638 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0);
6639 default: return Register();
6640 }
6641}
6642
6643// FastEmit functions for X86ISD::CVTTS2UIS.
6644
6645Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i32_r(Register Op0) {
6646 if ((Subtarget->hasAVX10_2())) {
6647 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USISrr_Int, RC: &X86::GR32RegClass, Op0);
6648 }
6649 return Register();
6650}
6651
6652Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i64_r(Register Op0) {
6653 if ((Subtarget->hasAVX10_2())) {
6654 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6655 }
6656 return Register();
6657}
6658
6659Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6660switch (RetVT.SimpleTy) {
6661 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i32_r(Op0);
6662 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i64_r(Op0);
6663 default: return Register();
6664}
6665}
6666
6667Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i32_r(Register Op0) {
6668 if ((Subtarget->hasAVX10_2())) {
6669 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USISrr_Int, RC: &X86::GR32RegClass, Op0);
6670 }
6671 return Register();
6672}
6673
6674Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i64_r(Register Op0) {
6675 if ((Subtarget->hasAVX10_2())) {
6676 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6677 }
6678 return Register();
6679}
6680
6681Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6682switch (RetVT.SimpleTy) {
6683 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i32_r(Op0);
6684 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i64_r(Op0);
6685 default: return Register();
6686}
6687}
6688
6689Register fastEmit_X86ISD_CVTTS2UIS_r(MVT VT, MVT RetVT, Register Op0) {
6690 switch (VT.SimpleTy) {
6691 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_r(RetVT, Op0);
6692 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_r(RetVT, Op0);
6693 default: return Register();
6694 }
6695}
6696
6697// FastEmit functions for X86ISD::CVTTS2UIS_SAE.
6698
6699Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6700 if ((Subtarget->hasAVX10_2())) {
6701 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USISrrb_Int, RC: &X86::GR32RegClass, Op0);
6702 }
6703 return Register();
6704}
6705
6706Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6707 if ((Subtarget->hasAVX10_2())) {
6708 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6709 }
6710 return Register();
6711}
6712
6713Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6714switch (RetVT.SimpleTy) {
6715 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i32_r(Op0);
6716 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i64_r(Op0);
6717 default: return Register();
6718}
6719}
6720
6721Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6722 if ((Subtarget->hasAVX10_2())) {
6723 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USISrrb_Int, RC: &X86::GR32RegClass, Op0);
6724 }
6725 return Register();
6726}
6727
6728Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6729 if ((Subtarget->hasAVX10_2())) {
6730 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6731 }
6732 return Register();
6733}
6734
6735Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6736switch (RetVT.SimpleTy) {
6737 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i32_r(Op0);
6738 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i64_r(Op0);
6739 default: return Register();
6740}
6741}
6742
6743Register fastEmit_X86ISD_CVTTS2UIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6744 switch (VT.SimpleTy) {
6745 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_r(RetVT, Op0);
6746 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_r(RetVT, Op0);
6747 default: return Register();
6748 }
6749}
6750
6751// FastEmit functions for X86ISD::CVTTS2UI_SAE.
6752
6753Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Register Op0) {
6754 if ((Subtarget->hasFP16())) {
6755 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6756 }
6757 return Register();
6758}
6759
6760Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Register Op0) {
6761 if ((Subtarget->hasFP16())) {
6762 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6763 }
6764 return Register();
6765}
6766
6767Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6768switch (RetVT.SimpleTy) {
6769 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Op0);
6770 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Op0);
6771 default: return Register();
6772}
6773}
6774
6775Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6776 if ((Subtarget->hasAVX512())) {
6777 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6778 }
6779 return Register();
6780}
6781
6782Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6783 if ((Subtarget->hasAVX512())) {
6784 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6785 }
6786 return Register();
6787}
6788
6789Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6790switch (RetVT.SimpleTy) {
6791 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Op0);
6792 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Op0);
6793 default: return Register();
6794}
6795}
6796
6797Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6798 if ((Subtarget->hasAVX512())) {
6799 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6800 }
6801 return Register();
6802}
6803
6804Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6805 if ((Subtarget->hasAVX512())) {
6806 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6807 }
6808 return Register();
6809}
6810
6811Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6812switch (RetVT.SimpleTy) {
6813 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Op0);
6814 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Op0);
6815 default: return Register();
6816}
6817}
6818
6819Register fastEmit_X86ISD_CVTTS2UI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6820 switch (VT.SimpleTy) {
6821 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(RetVT, Op0);
6822 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(RetVT, Op0);
6823 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(RetVT, Op0);
6824 default: return Register();
6825 }
6826}
6827
6828// FastEmit functions for X86ISD::CVTUI2P.
6829
6830Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
6831 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6832 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6833 }
6834 return Register();
6835}
6836
6837Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
6838 if ((Subtarget->hasVLX())) {
6839 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
6840 }
6841 return Register();
6842}
6843
6844Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
6845switch (RetVT.SimpleTy) {
6846 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
6847 case MVT::v2f64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
6848 default: return Register();
6849}
6850}
6851
6852Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
6853 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6854 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6855 }
6856 return Register();
6857}
6858
6859Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
6860 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6861 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
6862 }
6863 return Register();
6864}
6865
6866Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
6867switch (RetVT.SimpleTy) {
6868 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
6869 case MVT::v4f32: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
6870 default: return Register();
6871}
6872}
6873
6874Register fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
6875 if (RetVT.SimpleTy != MVT::v8f16)
6876 return Register();
6877 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6878 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
6879 }
6880 return Register();
6881}
6882
6883Register fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, Register Op0) {
6884 switch (VT.SimpleTy) {
6885 case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
6886 case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
6887 case MVT::v4i64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
6888 default: return Register();
6889 }
6890}
6891
6892// FastEmit functions for X86ISD::DYN_ALLOCA.
6893
6894Register fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
6895 if (RetVT.SimpleTy != MVT::isVoid)
6896 return Register();
6897 if ((!Subtarget->isTarget64BitLP64())) {
6898 return fastEmitInst_r(MachineInstOpcode: X86::DYN_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
6899 }
6900 return Register();
6901}
6902
6903Register fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
6904 if (RetVT.SimpleTy != MVT::isVoid)
6905 return Register();
6906 if ((Subtarget->is64Bit())) {
6907 return fastEmitInst_r(MachineInstOpcode: X86::DYN_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
6908 }
6909 return Register();
6910}
6911
6912Register fastEmit_X86ISD_DYN_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
6913 switch (VT.SimpleTy) {
6914 case MVT::i32: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(RetVT, Op0);
6915 case MVT::i64: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(RetVT, Op0);
6916 default: return Register();
6917 }
6918}
6919
6920// FastEmit functions for X86ISD::EH_RETURN.
6921
6922Register fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, Register Op0) {
6923 if (RetVT.SimpleTy != MVT::isVoid)
6924 return Register();
6925 return fastEmitInst_r(MachineInstOpcode: X86::EH_RETURN, RC: &X86::GR32RegClass, Op0);
6926}
6927
6928Register fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, Register Op0) {
6929 if (RetVT.SimpleTy != MVT::isVoid)
6930 return Register();
6931 return fastEmitInst_r(MachineInstOpcode: X86::EH_RETURN64, RC: &X86::GR64RegClass, Op0);
6932}
6933
6934Register fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, Register Op0) {
6935 switch (VT.SimpleTy) {
6936 case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0);
6937 case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0);
6938 default: return Register();
6939 }
6940}
6941
6942// FastEmit functions for X86ISD::FGETEXP.
6943
6944Register fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(MVT RetVT, Register Op0) {
6945 if (RetVT.SimpleTy != MVT::v8f16)
6946 return Register();
6947 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6948 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ128r, RC: &X86::VR128XRegClass, Op0);
6949 }
6950 return Register();
6951}
6952
6953Register fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(MVT RetVT, Register Op0) {
6954 if (RetVT.SimpleTy != MVT::v16f16)
6955 return Register();
6956 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6957 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ256r, RC: &X86::VR256XRegClass, Op0);
6958 }
6959 return Register();
6960}
6961
6962Register fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(MVT RetVT, Register Op0) {
6963 if (RetVT.SimpleTy != MVT::v32f16)
6964 return Register();
6965 if ((Subtarget->hasFP16())) {
6966 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZr, RC: &X86::VR512RegClass, Op0);
6967 }
6968 return Register();
6969}
6970
6971Register fastEmit_X86ISD_FGETEXP_MVT_v8bf16_r(MVT RetVT, Register Op0) {
6972 if (RetVT.SimpleTy != MVT::v8bf16)
6973 return Register();
6974 if ((Subtarget->hasAVX10_2())) {
6975 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Z128r, RC: &X86::VR128XRegClass, Op0);
6976 }
6977 return Register();
6978}
6979
6980Register fastEmit_X86ISD_FGETEXP_MVT_v16bf16_r(MVT RetVT, Register Op0) {
6981 if (RetVT.SimpleTy != MVT::v16bf16)
6982 return Register();
6983 if ((Subtarget->hasAVX10_2())) {
6984 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Z256r, RC: &X86::VR256XRegClass, Op0);
6985 }
6986 return Register();
6987}
6988
6989Register fastEmit_X86ISD_FGETEXP_MVT_v32bf16_r(MVT RetVT, Register Op0) {
6990 if (RetVT.SimpleTy != MVT::v32bf16)
6991 return Register();
6992 if ((Subtarget->hasAVX10_2())) {
6993 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Zr, RC: &X86::VR512RegClass, Op0);
6994 }
6995 return Register();
6996}
6997
6998Register fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(MVT RetVT, Register Op0) {
6999 if (RetVT.SimpleTy != MVT::v4f32)
7000 return Register();
7001 if ((Subtarget->hasVLX())) {
7002 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ128r, RC: &X86::VR128XRegClass, Op0);
7003 }
7004 return Register();
7005}
7006
7007Register fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7008 if (RetVT.SimpleTy != MVT::v8f32)
7009 return Register();
7010 if ((Subtarget->hasVLX())) {
7011 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ256r, RC: &X86::VR256XRegClass, Op0);
7012 }
7013 return Register();
7014}
7015
7016Register fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7017 if (RetVT.SimpleTy != MVT::v16f32)
7018 return Register();
7019 if ((Subtarget->hasAVX512())) {
7020 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZr, RC: &X86::VR512RegClass, Op0);
7021 }
7022 return Register();
7023}
7024
7025Register fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(MVT RetVT, Register Op0) {
7026 if (RetVT.SimpleTy != MVT::v2f64)
7027 return Register();
7028 if ((Subtarget->hasVLX())) {
7029 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ128r, RC: &X86::VR128XRegClass, Op0);
7030 }
7031 return Register();
7032}
7033
7034Register fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(MVT RetVT, Register Op0) {
7035 if (RetVT.SimpleTy != MVT::v4f64)
7036 return Register();
7037 if ((Subtarget->hasVLX())) {
7038 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ256r, RC: &X86::VR256XRegClass, Op0);
7039 }
7040 return Register();
7041}
7042
7043Register fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(MVT RetVT, Register Op0) {
7044 if (RetVT.SimpleTy != MVT::v8f64)
7045 return Register();
7046 if ((Subtarget->hasAVX512())) {
7047 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZr, RC: &X86::VR512RegClass, Op0);
7048 }
7049 return Register();
7050}
7051
7052Register fastEmit_X86ISD_FGETEXP_r(MVT VT, MVT RetVT, Register Op0) {
7053 switch (VT.SimpleTy) {
7054 case MVT::v8f16: return fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(RetVT, Op0);
7055 case MVT::v16f16: return fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(RetVT, Op0);
7056 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(RetVT, Op0);
7057 case MVT::v8bf16: return fastEmit_X86ISD_FGETEXP_MVT_v8bf16_r(RetVT, Op0);
7058 case MVT::v16bf16: return fastEmit_X86ISD_FGETEXP_MVT_v16bf16_r(RetVT, Op0);
7059 case MVT::v32bf16: return fastEmit_X86ISD_FGETEXP_MVT_v32bf16_r(RetVT, Op0);
7060 case MVT::v4f32: return fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(RetVT, Op0);
7061 case MVT::v8f32: return fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(RetVT, Op0);
7062 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(RetVT, Op0);
7063 case MVT::v2f64: return fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(RetVT, Op0);
7064 case MVT::v4f64: return fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(RetVT, Op0);
7065 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(RetVT, Op0);
7066 default: return Register();
7067 }
7068}
7069
7070// FastEmit functions for X86ISD::FGETEXP_SAE.
7071
7072Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
7073 if (RetVT.SimpleTy != MVT::v32f16)
7074 return Register();
7075 if ((Subtarget->hasFP16())) {
7076 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZrb, RC: &X86::VR512RegClass, Op0);
7077 }
7078 return Register();
7079}
7080
7081Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
7082 if (RetVT.SimpleTy != MVT::v16f32)
7083 return Register();
7084 if ((Subtarget->hasAVX512())) {
7085 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZrb, RC: &X86::VR512RegClass, Op0);
7086 }
7087 return Register();
7088}
7089
7090Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
7091 if (RetVT.SimpleTy != MVT::v8f64)
7092 return Register();
7093 if ((Subtarget->hasAVX512())) {
7094 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZrb, RC: &X86::VR512RegClass, Op0);
7095 }
7096 return Register();
7097}
7098
7099Register fastEmit_X86ISD_FGETEXP_SAE_r(MVT VT, MVT RetVT, Register Op0) {
7100 switch (VT.SimpleTy) {
7101 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(RetVT, Op0);
7102 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(RetVT, Op0);
7103 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(RetVT, Op0);
7104 default: return Register();
7105 }
7106}
7107
7108// FastEmit functions for X86ISD::FP_TO_SINT_SAT.
7109
7110Register fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7111 if (RetVT.SimpleTy != MVT::v2i64)
7112 return Register();
7113 if ((Subtarget->hasAVX10_2())) {
7114 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7115 }
7116 return Register();
7117}
7118
7119Register fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
7120 if (RetVT.SimpleTy != MVT::v4i32)
7121 return Register();
7122 if ((Subtarget->hasAVX10_2())) {
7123 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7124 }
7125 return Register();
7126}
7127
7128Register fastEmit_X86ISD_FP_TO_SINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
7129 switch (VT.SimpleTy) {
7130 case MVT::v4f32: return fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v4f32_r(RetVT, Op0);
7131 case MVT::v2f64: return fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v2f64_r(RetVT, Op0);
7132 default: return Register();
7133 }
7134}
7135
7136// FastEmit functions for X86ISD::FP_TO_UINT_SAT.
7137
7138Register fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7139 if (RetVT.SimpleTy != MVT::v2i64)
7140 return Register();
7141 if ((Subtarget->hasAVX10_2())) {
7142 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7143 }
7144 return Register();
7145}
7146
7147Register fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
7148 if (RetVT.SimpleTy != MVT::v4i32)
7149 return Register();
7150 if ((Subtarget->hasAVX10_2())) {
7151 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7152 }
7153 return Register();
7154}
7155
7156Register fastEmit_X86ISD_FP_TO_UINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
7157 switch (VT.SimpleTy) {
7158 case MVT::v4f32: return fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v4f32_r(RetVT, Op0);
7159 case MVT::v2f64: return fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v2f64_r(RetVT, Op0);
7160 default: return Register();
7161 }
7162}
7163
7164// FastEmit functions for X86ISD::FRCP.
7165
7166Register fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, Register Op0) {
7167 if (RetVT.SimpleTy != MVT::f32)
7168 return Register();
7169 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7170 return fastEmitInst_r(MachineInstOpcode: X86::RCPSSr, RC: &X86::FR32RegClass, Op0);
7171 }
7172 return Register();
7173}
7174
7175Register fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7176 if (RetVT.SimpleTy != MVT::v4f32)
7177 return Register();
7178 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7179 return fastEmitInst_r(MachineInstOpcode: X86::RCPPSr, RC: &X86::VR128RegClass, Op0);
7180 }
7181 if ((Subtarget->hasAVX())) {
7182 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSr, RC: &X86::VR128RegClass, Op0);
7183 }
7184 return Register();
7185}
7186
7187Register fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7188 if (RetVT.SimpleTy != MVT::v8f32)
7189 return Register();
7190 if ((Subtarget->hasAVX())) {
7191 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSYr, RC: &X86::VR256RegClass, Op0);
7192 }
7193 return Register();
7194}
7195
7196Register fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, Register Op0) {
7197 switch (VT.SimpleTy) {
7198 case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0);
7199 case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0);
7200 case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0);
7201 default: return Register();
7202 }
7203}
7204
7205// FastEmit functions for X86ISD::FRSQRT.
7206
7207Register fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
7208 if (RetVT.SimpleTy != MVT::f32)
7209 return Register();
7210 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7211 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTSSr, RC: &X86::FR32RegClass, Op0);
7212 }
7213 return Register();
7214}
7215
7216Register fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7217 if (RetVT.SimpleTy != MVT::v4f32)
7218 return Register();
7219 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7220 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTPSr, RC: &X86::VR128RegClass, Op0);
7221 }
7222 if ((Subtarget->hasAVX())) {
7223 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSr, RC: &X86::VR128RegClass, Op0);
7224 }
7225 return Register();
7226}
7227
7228Register fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
7229 if (RetVT.SimpleTy != MVT::v8f32)
7230 return Register();
7231 if ((Subtarget->hasAVX())) {
7232 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSYr, RC: &X86::VR256RegClass, Op0);
7233 }
7234 return Register();
7235}
7236
7237Register fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, Register Op0) {
7238 switch (VT.SimpleTy) {
7239 case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0);
7240 case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0);
7241 case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0);
7242 default: return Register();
7243 }
7244}
7245
7246// FastEmit functions for X86ISD::MMX_MOVD2W.
7247
7248Register fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, Register Op0) {
7249 if (RetVT.SimpleTy != MVT::i32)
7250 return Register();
7251 if ((Subtarget->hasMMX())) {
7252 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64grr, RC: &X86::GR32RegClass, Op0);
7253 }
7254 return Register();
7255}
7256
7257Register fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, Register Op0) {
7258 switch (VT.SimpleTy) {
7259 case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0);
7260 default: return Register();
7261 }
7262}
7263
7264// FastEmit functions for X86ISD::MMX_MOVW2D.
7265
7266Register fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, Register Op0) {
7267 if (RetVT.SimpleTy != MVT::x86mmx)
7268 return Register();
7269 if ((Subtarget->hasMMX())) {
7270 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64rr, RC: &X86::VR64RegClass, Op0);
7271 }
7272 return Register();
7273}
7274
7275Register fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, Register Op0) {
7276 switch (VT.SimpleTy) {
7277 case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0);
7278 default: return Register();
7279 }
7280}
7281
7282// FastEmit functions for X86ISD::MOVDDUP.
7283
7284Register fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, Register Op0) {
7285 if (RetVT.SimpleTy != MVT::v2f64)
7286 return Register();
7287 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7288 return fastEmitInst_r(MachineInstOpcode: X86::MOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7289 }
7290 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7291 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7292 }
7293 return Register();
7294}
7295
7296Register fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, Register Op0) {
7297 if (RetVT.SimpleTy != MVT::v4f64)
7298 return Register();
7299 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7300 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7301 }
7302 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7303 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPYrr, RC: &X86::VR256RegClass, Op0);
7304 }
7305 return Register();
7306}
7307
7308Register fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, Register Op0) {
7309 if (RetVT.SimpleTy != MVT::v8f64)
7310 return Register();
7311 if ((Subtarget->hasAVX512())) {
7312 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZrr, RC: &X86::VR512RegClass, Op0);
7313 }
7314 return Register();
7315}
7316
7317Register fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, Register Op0) {
7318 switch (VT.SimpleTy) {
7319 case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0);
7320 case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0);
7321 case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0);
7322 default: return Register();
7323 }
7324}
7325
7326// FastEmit functions for X86ISD::MOVDQ2Q.
7327
7328Register fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, Register Op0) {
7329 if (RetVT.SimpleTy != MVT::x86mmx)
7330 return Register();
7331 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
7332 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVDQ2Qrr, RC: &X86::VR64RegClass, Op0);
7333 }
7334 return Register();
7335}
7336
7337Register fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, Register Op0) {
7338 switch (VT.SimpleTy) {
7339 case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0);
7340 default: return Register();
7341 }
7342}
7343
7344// FastEmit functions for X86ISD::MOVMSK.
7345
7346Register fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, Register Op0) {
7347 if (RetVT.SimpleTy != MVT::i32)
7348 return Register();
7349 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7350 return fastEmitInst_r(MachineInstOpcode: X86::PMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
7351 }
7352 if ((Subtarget->hasAVX())) {
7353 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
7354 }
7355 return Register();
7356}
7357
7358Register fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, Register Op0) {
7359 if (RetVT.SimpleTy != MVT::i32)
7360 return Register();
7361 if ((Subtarget->hasAVX2())) {
7362 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBYrr, RC: &X86::GR32RegClass, Op0);
7363 }
7364 return Register();
7365}
7366
7367Register fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, Register Op0) {
7368 if (RetVT.SimpleTy != MVT::i32)
7369 return Register();
7370 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7371 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7372 }
7373 if ((Subtarget->hasAVX())) {
7374 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7375 }
7376 return Register();
7377}
7378
7379Register fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, Register Op0) {
7380 if (RetVT.SimpleTy != MVT::i32)
7381 return Register();
7382 if ((Subtarget->hasAVX())) {
7383 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
7384 }
7385 return Register();
7386}
7387
7388Register fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, Register Op0) {
7389 if (RetVT.SimpleTy != MVT::i32)
7390 return Register();
7391 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7392 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7393 }
7394 if ((Subtarget->hasAVX())) {
7395 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7396 }
7397 return Register();
7398}
7399
7400Register fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, Register Op0) {
7401 if (RetVT.SimpleTy != MVT::i32)
7402 return Register();
7403 if ((Subtarget->hasAVX())) {
7404 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
7405 }
7406 return Register();
7407}
7408
7409Register fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, Register Op0) {
7410 if (RetVT.SimpleTy != MVT::i32)
7411 return Register();
7412 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7413 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7414 }
7415 if ((Subtarget->hasAVX())) {
7416 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7417 }
7418 return Register();
7419}
7420
7421Register fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, Register Op0) {
7422 if (RetVT.SimpleTy != MVT::i32)
7423 return Register();
7424 if ((Subtarget->hasAVX())) {
7425 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
7426 }
7427 return Register();
7428}
7429
7430Register fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, Register Op0) {
7431 if (RetVT.SimpleTy != MVT::i32)
7432 return Register();
7433 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7434 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7435 }
7436 if ((Subtarget->hasAVX())) {
7437 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7438 }
7439 return Register();
7440}
7441
7442Register fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, Register Op0) {
7443 if (RetVT.SimpleTy != MVT::i32)
7444 return Register();
7445 if ((Subtarget->hasAVX())) {
7446 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
7447 }
7448 return Register();
7449}
7450
7451Register fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, Register Op0) {
7452 switch (VT.SimpleTy) {
7453 case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0);
7454 case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0);
7455 case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0);
7456 case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0);
7457 case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0);
7458 case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0);
7459 case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0);
7460 case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0);
7461 case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0);
7462 case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0);
7463 default: return Register();
7464 }
7465}
7466
7467// FastEmit functions for X86ISD::MOVQ2DQ.
7468
7469Register fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(MVT RetVT, Register Op0) {
7470 if (RetVT.SimpleTy != MVT::v2i64)
7471 return Register();
7472 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
7473 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2DQrr, RC: &X86::VR128RegClass, Op0);
7474 }
7475 return Register();
7476}
7477
7478Register fastEmit_X86ISD_MOVQ2DQ_r(MVT VT, MVT RetVT, Register Op0) {
7479 switch (VT.SimpleTy) {
7480 case MVT::x86mmx: return fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(RetVT, Op0);
7481 default: return Register();
7482 }
7483}
7484
7485// FastEmit functions for X86ISD::MOVSHDUP.
7486
7487Register fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, Register Op0) {
7488 if (RetVT.SimpleTy != MVT::v4i32)
7489 return Register();
7490 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7491 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7492 }
7493 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7494 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7495 }
7496 return Register();
7497}
7498
7499Register fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, Register Op0) {
7500 if (RetVT.SimpleTy != MVT::v8i32)
7501 return Register();
7502 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7503 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
7504 }
7505 return Register();
7506}
7507
7508Register fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7509 if (RetVT.SimpleTy != MVT::v4f32)
7510 return Register();
7511 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7512 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7513 }
7514 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7515 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7516 }
7517 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7518 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7519 }
7520 return Register();
7521}
7522
7523Register fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7524 if (RetVT.SimpleTy != MVT::v8f32)
7525 return Register();
7526 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7527 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7528 }
7529 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7530 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
7531 }
7532 return Register();
7533}
7534
7535Register fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7536 if (RetVT.SimpleTy != MVT::v16f32)
7537 return Register();
7538 if ((Subtarget->hasAVX512())) {
7539 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZrr, RC: &X86::VR512RegClass, Op0);
7540 }
7541 return Register();
7542}
7543
7544Register fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, Register Op0) {
7545 switch (VT.SimpleTy) {
7546 case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0);
7547 case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0);
7548 case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0);
7549 case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0);
7550 case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0);
7551 default: return Register();
7552 }
7553}
7554
7555// FastEmit functions for X86ISD::MOVSLDUP.
7556
7557Register fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, Register Op0) {
7558 if (RetVT.SimpleTy != MVT::v4i32)
7559 return Register();
7560 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7561 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7562 }
7563 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7564 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7565 }
7566 return Register();
7567}
7568
7569Register fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, Register Op0) {
7570 if (RetVT.SimpleTy != MVT::v8i32)
7571 return Register();
7572 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7573 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
7574 }
7575 return Register();
7576}
7577
7578Register fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7579 if (RetVT.SimpleTy != MVT::v4f32)
7580 return Register();
7581 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7582 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7583 }
7584 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7585 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7586 }
7587 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7588 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7589 }
7590 return Register();
7591}
7592
7593Register fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7594 if (RetVT.SimpleTy != MVT::v8f32)
7595 return Register();
7596 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7597 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7598 }
7599 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7600 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
7601 }
7602 return Register();
7603}
7604
7605Register fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7606 if (RetVT.SimpleTy != MVT::v16f32)
7607 return Register();
7608 if ((Subtarget->hasAVX512())) {
7609 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZrr, RC: &X86::VR512RegClass, Op0);
7610 }
7611 return Register();
7612}
7613
7614Register fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, Register Op0) {
7615 switch (VT.SimpleTy) {
7616 case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0);
7617 case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0);
7618 case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0);
7619 case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0);
7620 case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0);
7621 default: return Register();
7622 }
7623}
7624
7625// FastEmit functions for X86ISD::NT_BRIND.
7626
7627Register fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, Register Op0) {
7628 if (RetVT.SimpleTy != MVT::isVoid)
7629 return Register();
7630 if ((!Subtarget->is64Bit())) {
7631 return fastEmitInst_r(MachineInstOpcode: X86::JMP16r_NT, RC: &X86::GR16RegClass, Op0);
7632 }
7633 return Register();
7634}
7635
7636Register fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
7637 if (RetVT.SimpleTy != MVT::isVoid)
7638 return Register();
7639 if ((!Subtarget->is64Bit())) {
7640 return fastEmitInst_r(MachineInstOpcode: X86::JMP32r_NT, RC: &X86::GR32RegClass, Op0);
7641 }
7642 return Register();
7643}
7644
7645Register fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
7646 if (RetVT.SimpleTy != MVT::isVoid)
7647 return Register();
7648 if ((Subtarget->is64Bit())) {
7649 return fastEmitInst_r(MachineInstOpcode: X86::JMP64r_NT, RC: &X86::GR64RegClass, Op0);
7650 }
7651 return Register();
7652}
7653
7654Register fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
7655 switch (VT.SimpleTy) {
7656 case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0);
7657 case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0);
7658 case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0);
7659 default: return Register();
7660 }
7661}
7662
7663// FastEmit functions for X86ISD::NT_CALL.
7664
7665Register fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, Register Op0) {
7666 if (RetVT.SimpleTy != MVT::isVoid)
7667 return Register();
7668 if ((!Subtarget->is64Bit())) {
7669 return fastEmitInst_r(MachineInstOpcode: X86::CALL16r_NT, RC: &X86::GR16RegClass, Op0);
7670 }
7671 return Register();
7672}
7673
7674Register fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
7675 if (RetVT.SimpleTy != MVT::isVoid)
7676 return Register();
7677 if ((!Subtarget->is64Bit())) {
7678 return fastEmitInst_r(MachineInstOpcode: X86::CALL32r_NT, RC: &X86::GR32RegClass, Op0);
7679 }
7680 return Register();
7681}
7682
7683Register fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
7684 if (RetVT.SimpleTy != MVT::isVoid)
7685 return Register();
7686 if ((Subtarget->is64Bit())) {
7687 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r_NT, RC: &X86::GR64RegClass, Op0);
7688 }
7689 return Register();
7690}
7691
7692Register fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, Register Op0) {
7693 switch (VT.SimpleTy) {
7694 case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0);
7695 case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0);
7696 case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0);
7697 default: return Register();
7698 }
7699}
7700
7701// FastEmit functions for X86ISD::PHMINPOS.
7702
7703Register fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, Register Op0) {
7704 if (RetVT.SimpleTy != MVT::v8i16)
7705 return Register();
7706 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
7707 return fastEmitInst_r(MachineInstOpcode: X86::PHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
7708 }
7709 if ((Subtarget->hasAVX())) {
7710 return fastEmitInst_r(MachineInstOpcode: X86::VPHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
7711 }
7712 return Register();
7713}
7714
7715Register fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, Register Op0) {
7716 switch (VT.SimpleTy) {
7717 case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0);
7718 default: return Register();
7719 }
7720}
7721
7722// FastEmit functions for X86ISD::PROBED_ALLOCA.
7723
7724Register fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
7725 if (RetVT.SimpleTy != MVT::i32)
7726 return Register();
7727 if ((!Subtarget->isTarget64BitLP64())) {
7728 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
7729 }
7730 return Register();
7731}
7732
7733Register fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
7734 if (RetVT.SimpleTy != MVT::i64)
7735 return Register();
7736 if ((Subtarget->is64Bit())) {
7737 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
7738 }
7739 return Register();
7740}
7741
7742Register fastEmit_X86ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
7743 switch (VT.SimpleTy) {
7744 case MVT::i32: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(RetVT, Op0);
7745 case MVT::i64: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
7746 default: return Register();
7747 }
7748}
7749
7750// FastEmit functions for X86ISD::RCP14.
7751
7752Register fastEmit_X86ISD_RCP14_MVT_v8f16_r(MVT RetVT, Register Op0) {
7753 if (RetVT.SimpleTy != MVT::v8f16)
7754 return Register();
7755 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7756 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ128r, RC: &X86::VR128XRegClass, Op0);
7757 }
7758 return Register();
7759}
7760
7761Register fastEmit_X86ISD_RCP14_MVT_v16f16_r(MVT RetVT, Register Op0) {
7762 if (RetVT.SimpleTy != MVT::v16f16)
7763 return Register();
7764 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7765 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ256r, RC: &X86::VR256XRegClass, Op0);
7766 }
7767 return Register();
7768}
7769
7770Register fastEmit_X86ISD_RCP14_MVT_v32f16_r(MVT RetVT, Register Op0) {
7771 if (RetVT.SimpleTy != MVT::v32f16)
7772 return Register();
7773 if ((Subtarget->hasFP16())) {
7774 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZr, RC: &X86::VR512RegClass, Op0);
7775 }
7776 return Register();
7777}
7778
7779Register fastEmit_X86ISD_RCP14_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7780 if (RetVT.SimpleTy != MVT::v8bf16)
7781 return Register();
7782 if ((Subtarget->hasAVX10_2())) {
7783 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7784 }
7785 return Register();
7786}
7787
7788Register fastEmit_X86ISD_RCP14_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7789 if (RetVT.SimpleTy != MVT::v16bf16)
7790 return Register();
7791 if ((Subtarget->hasAVX10_2())) {
7792 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7793 }
7794 return Register();
7795}
7796
7797Register fastEmit_X86ISD_RCP14_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7798 if (RetVT.SimpleTy != MVT::v32bf16)
7799 return Register();
7800 if ((Subtarget->hasAVX10_2())) {
7801 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Zr, RC: &X86::VR512RegClass, Op0);
7802 }
7803 return Register();
7804}
7805
7806Register fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, Register Op0) {
7807 if (RetVT.SimpleTy != MVT::v4f32)
7808 return Register();
7809 if ((Subtarget->hasVLX())) {
7810 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ128r, RC: &X86::VR128XRegClass, Op0);
7811 }
7812 return Register();
7813}
7814
7815Register fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, Register Op0) {
7816 if (RetVT.SimpleTy != MVT::v8f32)
7817 return Register();
7818 if ((Subtarget->hasVLX())) {
7819 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ256r, RC: &X86::VR256XRegClass, Op0);
7820 }
7821 return Register();
7822}
7823
7824Register fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, Register Op0) {
7825 if (RetVT.SimpleTy != MVT::v16f32)
7826 return Register();
7827 if ((Subtarget->hasAVX512())) {
7828 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZr, RC: &X86::VR512RegClass, Op0);
7829 }
7830 return Register();
7831}
7832
7833Register fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, Register Op0) {
7834 if (RetVT.SimpleTy != MVT::v2f64)
7835 return Register();
7836 if ((Subtarget->hasVLX())) {
7837 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ128r, RC: &X86::VR128XRegClass, Op0);
7838 }
7839 return Register();
7840}
7841
7842Register fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, Register Op0) {
7843 if (RetVT.SimpleTy != MVT::v4f64)
7844 return Register();
7845 if ((Subtarget->hasVLX())) {
7846 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ256r, RC: &X86::VR256XRegClass, Op0);
7847 }
7848 return Register();
7849}
7850
7851Register fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, Register Op0) {
7852 if (RetVT.SimpleTy != MVT::v8f64)
7853 return Register();
7854 if ((Subtarget->hasAVX512())) {
7855 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZr, RC: &X86::VR512RegClass, Op0);
7856 }
7857 return Register();
7858}
7859
7860Register fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, Register Op0) {
7861 switch (VT.SimpleTy) {
7862 case MVT::v8f16: return fastEmit_X86ISD_RCP14_MVT_v8f16_r(RetVT, Op0);
7863 case MVT::v16f16: return fastEmit_X86ISD_RCP14_MVT_v16f16_r(RetVT, Op0);
7864 case MVT::v32f16: return fastEmit_X86ISD_RCP14_MVT_v32f16_r(RetVT, Op0);
7865 case MVT::v8bf16: return fastEmit_X86ISD_RCP14_MVT_v8bf16_r(RetVT, Op0);
7866 case MVT::v16bf16: return fastEmit_X86ISD_RCP14_MVT_v16bf16_r(RetVT, Op0);
7867 case MVT::v32bf16: return fastEmit_X86ISD_RCP14_MVT_v32bf16_r(RetVT, Op0);
7868 case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0);
7869 case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0);
7870 case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0);
7871 case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0);
7872 case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0);
7873 case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0);
7874 default: return Register();
7875 }
7876}
7877
7878// FastEmit functions for X86ISD::RSQRT14.
7879
7880Register fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(MVT RetVT, Register Op0) {
7881 if (RetVT.SimpleTy != MVT::v8f16)
7882 return Register();
7883 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7884 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
7885 }
7886 return Register();
7887}
7888
7889Register fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(MVT RetVT, Register Op0) {
7890 if (RetVT.SimpleTy != MVT::v16f16)
7891 return Register();
7892 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7893 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
7894 }
7895 return Register();
7896}
7897
7898Register fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(MVT RetVT, Register Op0) {
7899 if (RetVT.SimpleTy != MVT::v32f16)
7900 return Register();
7901 if ((Subtarget->hasFP16())) {
7902 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZr, RC: &X86::VR512RegClass, Op0);
7903 }
7904 return Register();
7905}
7906
7907Register fastEmit_X86ISD_RSQRT14_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7908 if (RetVT.SimpleTy != MVT::v8bf16)
7909 return Register();
7910 if ((Subtarget->hasAVX10_2())) {
7911 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7912 }
7913 return Register();
7914}
7915
7916Register fastEmit_X86ISD_RSQRT14_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7917 if (RetVT.SimpleTy != MVT::v16bf16)
7918 return Register();
7919 if ((Subtarget->hasAVX10_2())) {
7920 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7921 }
7922 return Register();
7923}
7924
7925Register fastEmit_X86ISD_RSQRT14_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7926 if (RetVT.SimpleTy != MVT::v32bf16)
7927 return Register();
7928 if ((Subtarget->hasAVX10_2())) {
7929 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
7930 }
7931 return Register();
7932}
7933
7934Register fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, Register Op0) {
7935 if (RetVT.SimpleTy != MVT::v4f32)
7936 return Register();
7937 if ((Subtarget->hasVLX())) {
7938 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ128r, RC: &X86::VR128XRegClass, Op0);
7939 }
7940 return Register();
7941}
7942
7943Register fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, Register Op0) {
7944 if (RetVT.SimpleTy != MVT::v8f32)
7945 return Register();
7946 if ((Subtarget->hasVLX())) {
7947 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ256r, RC: &X86::VR256XRegClass, Op0);
7948 }
7949 return Register();
7950}
7951
7952Register fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, Register Op0) {
7953 if (RetVT.SimpleTy != MVT::v16f32)
7954 return Register();
7955 if ((Subtarget->hasAVX512())) {
7956 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZr, RC: &X86::VR512RegClass, Op0);
7957 }
7958 return Register();
7959}
7960
7961Register fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, Register Op0) {
7962 if (RetVT.SimpleTy != MVT::v2f64)
7963 return Register();
7964 if ((Subtarget->hasVLX())) {
7965 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ128r, RC: &X86::VR128XRegClass, Op0);
7966 }
7967 return Register();
7968}
7969
7970Register fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, Register Op0) {
7971 if (RetVT.SimpleTy != MVT::v4f64)
7972 return Register();
7973 if ((Subtarget->hasVLX())) {
7974 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ256r, RC: &X86::VR256XRegClass, Op0);
7975 }
7976 return Register();
7977}
7978
7979Register fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, Register Op0) {
7980 if (RetVT.SimpleTy != MVT::v8f64)
7981 return Register();
7982 if ((Subtarget->hasAVX512())) {
7983 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZr, RC: &X86::VR512RegClass, Op0);
7984 }
7985 return Register();
7986}
7987
7988Register fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, Register Op0) {
7989 switch (VT.SimpleTy) {
7990 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(RetVT, Op0);
7991 case MVT::v16f16: return fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(RetVT, Op0);
7992 case MVT::v32f16: return fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(RetVT, Op0);
7993 case MVT::v8bf16: return fastEmit_X86ISD_RSQRT14_MVT_v8bf16_r(RetVT, Op0);
7994 case MVT::v16bf16: return fastEmit_X86ISD_RSQRT14_MVT_v16bf16_r(RetVT, Op0);
7995 case MVT::v32bf16: return fastEmit_X86ISD_RSQRT14_MVT_v32bf16_r(RetVT, Op0);
7996 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0);
7997 case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0);
7998 case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0);
7999 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0);
8000 case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0);
8001 case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0);
8002 default: return Register();
8003 }
8004}
8005
8006// FastEmit functions for X86ISD::SEG_ALLOCA.
8007
8008Register fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
8009 if (RetVT.SimpleTy != MVT::i32)
8010 return Register();
8011 if ((!Subtarget->isTarget64BitLP64())) {
8012 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
8013 }
8014 return Register();
8015}
8016
8017Register fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
8018 if (RetVT.SimpleTy != MVT::i64)
8019 return Register();
8020 if ((Subtarget->is64Bit())) {
8021 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
8022 }
8023 return Register();
8024}
8025
8026Register fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
8027 switch (VT.SimpleTy) {
8028 case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0);
8029 case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0);
8030 default: return Register();
8031 }
8032}
8033
8034// FastEmit functions for X86ISD::STRICT_CVTPH2PS.
8035
8036Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Register Op0) {
8037 if ((Subtarget->hasVLX())) {
8038 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8039 }
8040 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
8041 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
8042 }
8043 return Register();
8044}
8045
8046Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Register Op0) {
8047 if ((Subtarget->hasVLX())) {
8048 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
8049 }
8050 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
8051 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
8052 }
8053 return Register();
8054}
8055
8056Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(MVT RetVT, Register Op0) {
8057switch (RetVT.SimpleTy) {
8058 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
8059 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
8060 default: return Register();
8061}
8062}
8063
8064Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(MVT RetVT, Register Op0) {
8065 if (RetVT.SimpleTy != MVT::v16f32)
8066 return Register();
8067 if ((Subtarget->hasAVX512())) {
8068 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
8069 }
8070 return Register();
8071}
8072
8073Register fastEmit_X86ISD_STRICT_CVTPH2PS_r(MVT VT, MVT RetVT, Register Op0) {
8074 switch (VT.SimpleTy) {
8075 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
8076 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
8077 default: return Register();
8078 }
8079}
8080
8081// FastEmit functions for X86ISD::STRICT_CVTSI2P.
8082
8083Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
8084 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8085 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8086 }
8087 return Register();
8088}
8089
8090Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
8091 if ((Subtarget->hasVLX())) {
8092 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8093 }
8094 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8095 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
8096 }
8097 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8098 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
8099 }
8100 return Register();
8101}
8102
8103Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
8104switch (RetVT.SimpleTy) {
8105 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
8106 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
8107 default: return Register();
8108}
8109}
8110
8111Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
8112 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8113 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8114 }
8115 return Register();
8116}
8117
8118Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
8119 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8120 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8121 }
8122 return Register();
8123}
8124
8125Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
8126switch (RetVT.SimpleTy) {
8127 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
8128 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
8129 default: return Register();
8130}
8131}
8132
8133Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
8134 if (RetVT.SimpleTy != MVT::v8f16)
8135 return Register();
8136 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8137 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8138 }
8139 return Register();
8140}
8141
8142Register fastEmit_X86ISD_STRICT_CVTSI2P_r(MVT VT, MVT RetVT, Register Op0) {
8143 switch (VT.SimpleTy) {
8144 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
8145 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
8146 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
8147 default: return Register();
8148 }
8149}
8150
8151// FastEmit functions for X86ISD::STRICT_CVTTP2SI.
8152
8153Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
8154 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8155 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
8156 }
8157 return Register();
8158}
8159
8160Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
8161 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8162 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8163 }
8164 return Register();
8165}
8166
8167Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
8168 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8169 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
8170 }
8171 return Register();
8172}
8173
8174Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
8175 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8176 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8177 }
8178 return Register();
8179}
8180
8181Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
8182 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8183 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8184 }
8185 return Register();
8186}
8187
8188Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
8189 if ((Subtarget->hasFP16())) {
8190 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
8191 }
8192 return Register();
8193}
8194
8195Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
8196switch (RetVT.SimpleTy) {
8197 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
8198 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
8199 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
8200 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
8201 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
8202 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
8203 default: return Register();
8204}
8205}
8206
8207Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
8208 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8209 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
8210 }
8211 return Register();
8212}
8213
8214Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
8215 if ((Subtarget->hasFP16())) {
8216 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
8217 }
8218 return Register();
8219}
8220
8221Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
8222switch (RetVT.SimpleTy) {
8223 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
8224 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
8225 default: return Register();
8226}
8227}
8228
8229Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
8230 if (RetVT.SimpleTy != MVT::v32i16)
8231 return Register();
8232 if ((Subtarget->hasFP16())) {
8233 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
8234 }
8235 return Register();
8236}
8237
8238Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
8239 if ((Subtarget->hasVLX())) {
8240 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8241 }
8242 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8243 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
8244 }
8245 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8246 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
8247 }
8248 return Register();
8249}
8250
8251Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
8252 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8253 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8254 }
8255 return Register();
8256}
8257
8258Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
8259 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8260 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8261 }
8262 return Register();
8263}
8264
8265Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
8266switch (RetVT.SimpleTy) {
8267 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
8268 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
8269 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
8270 default: return Register();
8271}
8272}
8273
8274Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
8275 if ((Subtarget->hasVLX())) {
8276 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
8277 }
8278 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8279 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
8280 }
8281 return Register();
8282}
8283
8284Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
8285 if ((Subtarget->hasDQI())) {
8286 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
8287 }
8288 return Register();
8289}
8290
8291Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
8292switch (RetVT.SimpleTy) {
8293 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
8294 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
8295 default: return Register();
8296}
8297}
8298
8299Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
8300 if (RetVT.SimpleTy != MVT::v16i32)
8301 return Register();
8302 if ((Subtarget->hasAVX512())) {
8303 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
8304 }
8305 return Register();
8306}
8307
8308Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
8309 if ((Subtarget->hasVLX())) {
8310 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8311 }
8312 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8313 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
8314 }
8315 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8316 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
8317 }
8318 return Register();
8319}
8320
8321Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
8322 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8323 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8324 }
8325 return Register();
8326}
8327
8328Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
8329switch (RetVT.SimpleTy) {
8330 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
8331 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
8332 default: return Register();
8333}
8334}
8335
8336Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
8337 if ((Subtarget->hasVLX())) {
8338 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
8339 }
8340 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8341 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
8342 }
8343 return Register();
8344}
8345
8346Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
8347 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8349 }
8350 return Register();
8351}
8352
8353Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
8354switch (RetVT.SimpleTy) {
8355 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
8356 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
8357 default: return Register();
8358}
8359}
8360
8361Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
8362 if ((Subtarget->hasAVX512())) {
8363 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
8364 }
8365 return Register();
8366}
8367
8368Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
8369 if ((Subtarget->hasDQI())) {
8370 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
8371 }
8372 return Register();
8373}
8374
8375Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
8376switch (RetVT.SimpleTy) {
8377 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
8378 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
8379 default: return Register();
8380}
8381}
8382
8383Register fastEmit_X86ISD_STRICT_CVTTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
8384 switch (VT.SimpleTy) {
8385 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
8386 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
8387 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
8388 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
8389 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
8390 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
8391 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
8392 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
8393 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
8394 default: return Register();
8395 }
8396}
8397
8398// FastEmit functions for X86ISD::STRICT_CVTTP2UI.
8399
8400Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
8401 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8402 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
8403 }
8404 return Register();
8405}
8406
8407Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
8408 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8409 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8410 }
8411 return Register();
8412}
8413
8414Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
8415 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8416 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
8417 }
8418 return Register();
8419}
8420
8421Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
8422 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8423 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8424 }
8425 return Register();
8426}
8427
8428Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
8429 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8430 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8431 }
8432 return Register();
8433}
8434
8435Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
8436 if ((Subtarget->hasFP16())) {
8437 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
8438 }
8439 return Register();
8440}
8441
8442Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
8443switch (RetVT.SimpleTy) {
8444 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
8445 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
8446 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
8447 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
8448 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
8449 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
8450 default: return Register();
8451}
8452}
8453
8454Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
8455 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8456 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
8457 }
8458 return Register();
8459}
8460
8461Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
8462 if ((Subtarget->hasFP16())) {
8463 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
8464 }
8465 return Register();
8466}
8467
8468Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
8469switch (RetVT.SimpleTy) {
8470 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
8471 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
8472 default: return Register();
8473}
8474}
8475
8476Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
8477 if (RetVT.SimpleTy != MVT::v32i16)
8478 return Register();
8479 if ((Subtarget->hasFP16())) {
8480 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
8481 }
8482 return Register();
8483}
8484
8485Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
8486 if ((Subtarget->hasVLX())) {
8487 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8488 }
8489 return Register();
8490}
8491
8492Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
8493 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8495 }
8496 return Register();
8497}
8498
8499Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
8500 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8502 }
8503 return Register();
8504}
8505
8506Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
8507switch (RetVT.SimpleTy) {
8508 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
8509 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
8510 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
8511 default: return Register();
8512}
8513}
8514
8515Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
8516 if ((Subtarget->hasVLX())) {
8517 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
8518 }
8519 return Register();
8520}
8521
8522Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
8523 if ((Subtarget->hasDQI())) {
8524 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
8525 }
8526 return Register();
8527}
8528
8529Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
8530switch (RetVT.SimpleTy) {
8531 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
8532 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
8533 default: return Register();
8534}
8535}
8536
8537Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
8538 if (RetVT.SimpleTy != MVT::v16i32)
8539 return Register();
8540 if ((Subtarget->hasAVX512())) {
8541 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
8542 }
8543 return Register();
8544}
8545
8546Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
8547 if ((Subtarget->hasVLX())) {
8548 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8549 }
8550 return Register();
8551}
8552
8553Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
8554 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8556 }
8557 return Register();
8558}
8559
8560Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
8561switch (RetVT.SimpleTy) {
8562 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
8563 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
8564 default: return Register();
8565}
8566}
8567
8568Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
8569 if ((Subtarget->hasVLX())) {
8570 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
8571 }
8572 return Register();
8573}
8574
8575Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
8576 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8578 }
8579 return Register();
8580}
8581
8582Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
8583switch (RetVT.SimpleTy) {
8584 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
8585 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
8586 default: return Register();
8587}
8588}
8589
8590Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
8591 if ((Subtarget->hasAVX512())) {
8592 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
8593 }
8594 return Register();
8595}
8596
8597Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
8598 if ((Subtarget->hasDQI())) {
8599 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
8600 }
8601 return Register();
8602}
8603
8604Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
8605switch (RetVT.SimpleTy) {
8606 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
8607 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
8608 default: return Register();
8609}
8610}
8611
8612Register fastEmit_X86ISD_STRICT_CVTTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
8613 switch (VT.SimpleTy) {
8614 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
8615 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
8616 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
8617 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
8618 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
8619 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
8620 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
8621 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
8622 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
8623 default: return Register();
8624 }
8625}
8626
8627// FastEmit functions for X86ISD::STRICT_CVTUI2P.
8628
8629Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
8630 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8631 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8632 }
8633 return Register();
8634}
8635
8636Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
8637 if ((Subtarget->hasVLX())) {
8638 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8639 }
8640 return Register();
8641}
8642
8643Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
8644switch (RetVT.SimpleTy) {
8645 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
8646 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
8647 default: return Register();
8648}
8649}
8650
8651Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
8652 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8653 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8654 }
8655 return Register();
8656}
8657
8658Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
8659 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8660 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8661 }
8662 return Register();
8663}
8664
8665Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
8666switch (RetVT.SimpleTy) {
8667 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
8668 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
8669 default: return Register();
8670}
8671}
8672
8673Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
8674 if (RetVT.SimpleTy != MVT::v8f16)
8675 return Register();
8676 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8677 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8678 }
8679 return Register();
8680}
8681
8682Register fastEmit_X86ISD_STRICT_CVTUI2P_r(MVT VT, MVT RetVT, Register Op0) {
8683 switch (VT.SimpleTy) {
8684 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
8685 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
8686 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
8687 default: return Register();
8688 }
8689}
8690
8691// FastEmit functions for X86ISD::STRICT_VFPEXT.
8692
8693Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Register Op0) {
8694 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8695 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
8696 }
8697 return Register();
8698}
8699
8700Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Register Op0) {
8701 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8702 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8703 }
8704 return Register();
8705}
8706
8707Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Register Op0) {
8708 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8709 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
8710 }
8711 return Register();
8712}
8713
8714Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(MVT RetVT, Register Op0) {
8715switch (RetVT.SimpleTy) {
8716 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
8717 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
8718 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
8719 default: return Register();
8720}
8721}
8722
8723Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(MVT RetVT, Register Op0) {
8724 if (RetVT.SimpleTy != MVT::v2f64)
8725 return Register();
8726 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8727 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8728 }
8729 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8730 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
8731 }
8732 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8733 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
8734 }
8735 return Register();
8736}
8737
8738Register fastEmit_X86ISD_STRICT_VFPEXT_r(MVT VT, MVT RetVT, Register Op0) {
8739 switch (VT.SimpleTy) {
8740 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(RetVT, Op0);
8741 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(RetVT, Op0);
8742 default: return Register();
8743 }
8744}
8745
8746// FastEmit functions for X86ISD::STRICT_VFPROUND.
8747
8748Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
8749 if (RetVT.SimpleTy != MVT::v8f16)
8750 return Register();
8751 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8752 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
8753 }
8754 return Register();
8755}
8756
8757Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(MVT RetVT, Register Op0) {
8758 if (RetVT.SimpleTy != MVT::v8f16)
8759 return Register();
8760 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8761 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
8762 }
8763 return Register();
8764}
8765
8766Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(MVT RetVT, Register Op0) {
8767 if (RetVT.SimpleTy != MVT::v16f16)
8768 return Register();
8769 if ((Subtarget->hasFP16())) {
8770 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
8771 }
8772 return Register();
8773}
8774
8775Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Register Op0) {
8776 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8777 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8778 }
8779 return Register();
8780}
8781
8782Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Register Op0) {
8783 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8784 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8785 }
8786 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8787 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8788 }
8789 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8790 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8791 }
8792 return Register();
8793}
8794
8795Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
8796switch (RetVT.SimpleTy) {
8797 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
8798 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
8799 default: return Register();
8800}
8801}
8802
8803Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Register Op0) {
8804 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8805 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8806 }
8807 return Register();
8808}
8809
8810Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Register Op0) {
8811 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8812 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
8813 }
8814 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8815 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
8816 }
8817 return Register();
8818}
8819
8820Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(MVT RetVT, Register Op0) {
8821switch (RetVT.SimpleTy) {
8822 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
8823 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
8824 default: return Register();
8825}
8826}
8827
8828Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Register Op0) {
8829 if ((Subtarget->hasFP16())) {
8830 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
8831 }
8832 return Register();
8833}
8834
8835Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Register Op0) {
8836 if ((Subtarget->hasAVX512())) {
8837 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
8838 }
8839 return Register();
8840}
8841
8842Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(MVT RetVT, Register Op0) {
8843switch (RetVT.SimpleTy) {
8844 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
8845 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
8846 default: return Register();
8847}
8848}
8849
8850Register fastEmit_X86ISD_STRICT_VFPROUND_r(MVT VT, MVT RetVT, Register Op0) {
8851 switch (VT.SimpleTy) {
8852 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(RetVT, Op0);
8853 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(RetVT, Op0);
8854 case MVT::v16f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(RetVT, Op0);
8855 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(RetVT, Op0);
8856 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(RetVT, Op0);
8857 case MVT::v8f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(RetVT, Op0);
8858 default: return Register();
8859 }
8860}
8861
8862// FastEmit functions for X86ISD::VBROADCAST.
8863
8864Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Register Op0) {
8865 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8866 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ128rr, RC: &X86::VR128XRegClass, Op0);
8867 }
8868 return Register();
8869}
8870
8871Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Register Op0) {
8872 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8873 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ256rr, RC: &X86::VR256XRegClass, Op0);
8874 }
8875 return Register();
8876}
8877
8878Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Register Op0) {
8879 if ((Subtarget->hasAVX512())) {
8880 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZrr, RC: &X86::VR512RegClass, Op0);
8881 }
8882 return Register();
8883}
8884
8885Register fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, Register Op0) {
8886switch (RetVT.SimpleTy) {
8887 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0);
8888 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0);
8889 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0);
8890 default: return Register();
8891}
8892}
8893
8894Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Register Op0) {
8895 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8896 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ128rr, RC: &X86::VR128XRegClass, Op0);
8897 }
8898 return Register();
8899}
8900
8901Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Register Op0) {
8902 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8903 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ256rr, RC: &X86::VR256XRegClass, Op0);
8904 }
8905 return Register();
8906}
8907
8908Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Register Op0) {
8909 if ((Subtarget->hasAVX512())) {
8910 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZrr, RC: &X86::VR512RegClass, Op0);
8911 }
8912 return Register();
8913}
8914
8915Register fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, Register Op0) {
8916switch (RetVT.SimpleTy) {
8917 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0);
8918 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0);
8919 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0);
8920 default: return Register();
8921}
8922}
8923
8924Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Register Op0) {
8925 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8926 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ128rr, RC: &X86::VR128XRegClass, Op0);
8927 }
8928 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8929 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBrr, RC: &X86::VR128RegClass, Op0);
8930 }
8931 return Register();
8932}
8933
8934Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Register Op0) {
8935 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8936 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ256rr, RC: &X86::VR256XRegClass, Op0);
8937 }
8938 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8939 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBYrr, RC: &X86::VR256RegClass, Op0);
8940 }
8941 return Register();
8942}
8943
8944Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Register Op0) {
8945 if ((Subtarget->hasBWI())) {
8946 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZrr, RC: &X86::VR512RegClass, Op0);
8947 }
8948 return Register();
8949}
8950
8951Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
8952switch (RetVT.SimpleTy) {
8953 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0);
8954 case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0);
8955 case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0);
8956 default: return Register();
8957}
8958}
8959
8960Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Register Op0) {
8961 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8962 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
8963 }
8964 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8965 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
8966 }
8967 return Register();
8968}
8969
8970Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Register Op0) {
8971 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8972 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
8973 }
8974 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8975 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
8976 }
8977 return Register();
8978}
8979
8980Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Register Op0) {
8981 if ((Subtarget->hasBWI())) {
8982 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
8983 }
8984 return Register();
8985}
8986
8987Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
8988switch (RetVT.SimpleTy) {
8989 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0);
8990 case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0);
8991 case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0);
8992 default: return Register();
8993}
8994}
8995
8996Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Register Op0) {
8997 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8998 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ128rr, RC: &X86::VR128XRegClass, Op0);
8999 }
9000 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9001 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrr, RC: &X86::VR128RegClass, Op0);
9002 }
9003 return Register();
9004}
9005
9006Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Register Op0) {
9007 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9008 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ256rr, RC: &X86::VR256XRegClass, Op0);
9009 }
9010 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9011 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDYrr, RC: &X86::VR256RegClass, Op0);
9012 }
9013 return Register();
9014}
9015
9016Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Register Op0) {
9017 if ((Subtarget->hasAVX512())) {
9018 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZrr, RC: &X86::VR512RegClass, Op0);
9019 }
9020 return Register();
9021}
9022
9023Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
9024switch (RetVT.SimpleTy) {
9025 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0);
9026 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0);
9027 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0);
9028 default: return Register();
9029}
9030}
9031
9032Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Register Op0) {
9033 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9034 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ128rr, RC: &X86::VR128XRegClass, Op0);
9035 }
9036 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9037 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrr, RC: &X86::VR128RegClass, Op0);
9038 }
9039 return Register();
9040}
9041
9042Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Register Op0) {
9043 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9044 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ256rr, RC: &X86::VR256XRegClass, Op0);
9045 }
9046 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9047 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQYrr, RC: &X86::VR256RegClass, Op0);
9048 }
9049 return Register();
9050}
9051
9052Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Register Op0) {
9053 if ((Subtarget->hasAVX512())) {
9054 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZrr, RC: &X86::VR512RegClass, Op0);
9055 }
9056 return Register();
9057}
9058
9059Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
9060switch (RetVT.SimpleTy) {
9061 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0);
9062 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0);
9063 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0);
9064 default: return Register();
9065}
9066}
9067
9068Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Register Op0) {
9069 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9070 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
9071 }
9072 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9073 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
9074 }
9075 return Register();
9076}
9077
9078Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Register Op0) {
9079 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9080 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
9081 }
9082 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9083 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
9084 }
9085 return Register();
9086}
9087
9088Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Register Op0) {
9089 if ((Subtarget->hasBWI())) {
9090 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
9091 }
9092 return Register();
9093}
9094
9095Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
9096switch (RetVT.SimpleTy) {
9097 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Op0);
9098 case MVT::v16f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Op0);
9099 case MVT::v32f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Op0);
9100 default: return Register();
9101}
9102}
9103
9104Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Register Op0) {
9105 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9106 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
9107 }
9108 return Register();
9109}
9110
9111Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Register Op0) {
9112 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9113 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
9114 }
9115 return Register();
9116}
9117
9118Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Register Op0) {
9119 if ((Subtarget->hasBF16())) {
9120 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
9121 }
9122 return Register();
9123}
9124
9125Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
9126switch (RetVT.SimpleTy) {
9127 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Op0);
9128 case MVT::v16bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Op0);
9129 case MVT::v32bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Op0);
9130 default: return Register();
9131}
9132}
9133
9134Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Register Op0) {
9135 if ((Subtarget->hasVLX())) {
9136 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ128rr, RC: &X86::VR128XRegClass, Op0);
9137 }
9138 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9139 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSrr, RC: &X86::VR128RegClass, Op0);
9140 }
9141 return Register();
9142}
9143
9144Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Register Op0) {
9145 if ((Subtarget->hasVLX())) {
9146 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ256rr, RC: &X86::VR256XRegClass, Op0);
9147 }
9148 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9149 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSYrr, RC: &X86::VR256RegClass, Op0);
9150 }
9151 return Register();
9152}
9153
9154Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Register Op0) {
9155 if ((Subtarget->hasAVX512())) {
9156 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZrr, RC: &X86::VR512RegClass, Op0);
9157 }
9158 return Register();
9159}
9160
9161Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
9162switch (RetVT.SimpleTy) {
9163 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0);
9164 case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0);
9165 case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0);
9166 default: return Register();
9167}
9168}
9169
9170Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Register Op0) {
9171 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9172 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
9173 }
9174 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9175 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
9176 }
9177 return Register();
9178}
9179
9180Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Register Op0) {
9181 if ((Subtarget->hasVLX())) {
9182 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZ256rr, RC: &X86::VR256XRegClass, Op0);
9183 }
9184 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9185 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDYrr, RC: &X86::VR256RegClass, Op0);
9186 }
9187 return Register();
9188}
9189
9190Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Register Op0) {
9191 if ((Subtarget->hasAVX512())) {
9192 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZrr, RC: &X86::VR512RegClass, Op0);
9193 }
9194 return Register();
9195}
9196
9197Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
9198switch (RetVT.SimpleTy) {
9199 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0);
9200 case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0);
9201 case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0);
9202 default: return Register();
9203}
9204}
9205
9206Register fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, Register Op0) {
9207 switch (VT.SimpleTy) {
9208 case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0);
9209 case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0);
9210 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0);
9211 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0);
9212 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0);
9213 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0);
9214 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(RetVT, Op0);
9215 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(RetVT, Op0);
9216 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0);
9217 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0);
9218 default: return Register();
9219 }
9220}
9221
9222// FastEmit functions for X86ISD::VBROADCASTM.
9223
9224Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Register Op0) {
9225 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9226 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ128rr, RC: &X86::VR128XRegClass, Op0);
9227 }
9228 return Register();
9229}
9230
9231Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Register Op0) {
9232 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9233 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ256rr, RC: &X86::VR256XRegClass, Op0);
9234 }
9235 return Register();
9236}
9237
9238Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Register Op0) {
9239 if ((Subtarget->hasCDI())) {
9240 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZrr, RC: &X86::VR512RegClass, Op0);
9241 }
9242 return Register();
9243}
9244
9245Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, Register Op0) {
9246switch (RetVT.SimpleTy) {
9247 case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0);
9248 case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0);
9249 case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0);
9250 default: return Register();
9251}
9252}
9253
9254Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Register Op0) {
9255 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9256 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ128rr, RC: &X86::VR128XRegClass, Op0);
9257 }
9258 return Register();
9259}
9260
9261Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Register Op0) {
9262 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9263 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ256rr, RC: &X86::VR256XRegClass, Op0);
9264 }
9265 return Register();
9266}
9267
9268Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Register Op0) {
9269 if ((Subtarget->hasCDI())) {
9270 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZrr, RC: &X86::VR512RegClass, Op0);
9271 }
9272 return Register();
9273}
9274
9275Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, Register Op0) {
9276switch (RetVT.SimpleTy) {
9277 case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0);
9278 case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0);
9279 case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0);
9280 default: return Register();
9281}
9282}
9283
9284Register fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, Register Op0) {
9285 switch (VT.SimpleTy) {
9286 case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0);
9287 case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0);
9288 default: return Register();
9289 }
9290}
9291
9292// FastEmit functions for X86ISD::VCVTHF82PH.
9293
9294Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v8f16_r(Register Op0) {
9295 if ((Subtarget->hasAVX10_2())) {
9296 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZ128rr, RC: &X86::VR128XRegClass, Op0);
9297 }
9298 return Register();
9299}
9300
9301Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v16f16_r(Register Op0) {
9302 if ((Subtarget->hasAVX10_2())) {
9303 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZ256rr, RC: &X86::VR256XRegClass, Op0);
9304 }
9305 return Register();
9306}
9307
9308Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_r(MVT RetVT, Register Op0) {
9309switch (RetVT.SimpleTy) {
9310 case MVT::v8f16: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v8f16_r(Op0);
9311 case MVT::v16f16: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v16f16_r(Op0);
9312 default: return Register();
9313}
9314}
9315
9316Register fastEmit_X86ISD_VCVTHF82PH_MVT_v32i8_r(MVT RetVT, Register Op0) {
9317 if (RetVT.SimpleTy != MVT::v32f16)
9318 return Register();
9319 if ((Subtarget->hasAVX10_2())) {
9320 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZrr, RC: &X86::VR512RegClass, Op0);
9321 }
9322 return Register();
9323}
9324
9325Register fastEmit_X86ISD_VCVTHF82PH_r(MVT VT, MVT RetVT, Register Op0) {
9326 switch (VT.SimpleTy) {
9327 case MVT::v16i8: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_r(RetVT, Op0);
9328 case MVT::v32i8: return fastEmit_X86ISD_VCVTHF82PH_MVT_v32i8_r(RetVT, Op0);
9329 default: return Register();
9330 }
9331}
9332
9333// FastEmit functions for X86ISD::VCVTPH2BF8.
9334
9335Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v8f16_r(MVT RetVT, Register Op0) {
9336 if (RetVT.SimpleTy != MVT::v16i8)
9337 return Register();
9338 if ((Subtarget->hasAVX10_2())) {
9339 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9340 }
9341 return Register();
9342}
9343
9344Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v16f16_r(MVT RetVT, Register Op0) {
9345 if (RetVT.SimpleTy != MVT::v16i8)
9346 return Register();
9347 if ((Subtarget->hasAVX10_2())) {
9348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9349 }
9350 return Register();
9351}
9352
9353Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v32f16_r(MVT RetVT, Register Op0) {
9354 if (RetVT.SimpleTy != MVT::v32i8)
9355 return Register();
9356 if ((Subtarget->hasAVX10_2())) {
9357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Zrr, RC: &X86::VR256XRegClass, Op0);
9358 }
9359 return Register();
9360}
9361
9362Register fastEmit_X86ISD_VCVTPH2BF8_r(MVT VT, MVT RetVT, Register Op0) {
9363 switch (VT.SimpleTy) {
9364 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v8f16_r(RetVT, Op0);
9365 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v16f16_r(RetVT, Op0);
9366 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v32f16_r(RetVT, Op0);
9367 default: return Register();
9368 }
9369}
9370
9371// FastEmit functions for X86ISD::VCVTPH2BF8S.
9372
9373Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v8f16_r(MVT RetVT, Register Op0) {
9374 if (RetVT.SimpleTy != MVT::v16i8)
9375 return Register();
9376 if ((Subtarget->hasAVX10_2())) {
9377 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9378 }
9379 return Register();
9380}
9381
9382Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v16f16_r(MVT RetVT, Register Op0) {
9383 if (RetVT.SimpleTy != MVT::v16i8)
9384 return Register();
9385 if ((Subtarget->hasAVX10_2())) {
9386 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9387 }
9388 return Register();
9389}
9390
9391Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v32f16_r(MVT RetVT, Register Op0) {
9392 if (RetVT.SimpleTy != MVT::v32i8)
9393 return Register();
9394 if ((Subtarget->hasAVX10_2())) {
9395 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZrr, RC: &X86::VR256XRegClass, Op0);
9396 }
9397 return Register();
9398}
9399
9400Register fastEmit_X86ISD_VCVTPH2BF8S_r(MVT VT, MVT RetVT, Register Op0) {
9401 switch (VT.SimpleTy) {
9402 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v8f16_r(RetVT, Op0);
9403 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v16f16_r(RetVT, Op0);
9404 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v32f16_r(RetVT, Op0);
9405 default: return Register();
9406 }
9407}
9408
9409// FastEmit functions for X86ISD::VCVTPH2HF8.
9410
9411Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v8f16_r(MVT RetVT, Register Op0) {
9412 if (RetVT.SimpleTy != MVT::v16i8)
9413 return Register();
9414 if ((Subtarget->hasAVX10_2())) {
9415 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9416 }
9417 return Register();
9418}
9419
9420Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v16f16_r(MVT RetVT, Register Op0) {
9421 if (RetVT.SimpleTy != MVT::v16i8)
9422 return Register();
9423 if ((Subtarget->hasAVX10_2())) {
9424 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9425 }
9426 return Register();
9427}
9428
9429Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v32f16_r(MVT RetVT, Register Op0) {
9430 if (RetVT.SimpleTy != MVT::v32i8)
9431 return Register();
9432 if ((Subtarget->hasAVX10_2())) {
9433 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Zrr, RC: &X86::VR256XRegClass, Op0);
9434 }
9435 return Register();
9436}
9437
9438Register fastEmit_X86ISD_VCVTPH2HF8_r(MVT VT, MVT RetVT, Register Op0) {
9439 switch (VT.SimpleTy) {
9440 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v8f16_r(RetVT, Op0);
9441 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v16f16_r(RetVT, Op0);
9442 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v32f16_r(RetVT, Op0);
9443 default: return Register();
9444 }
9445}
9446
9447// FastEmit functions for X86ISD::VCVTPH2HF8S.
9448
9449Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v8f16_r(MVT RetVT, Register Op0) {
9450 if (RetVT.SimpleTy != MVT::v16i8)
9451 return Register();
9452 if ((Subtarget->hasAVX10_2())) {
9453 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9454 }
9455 return Register();
9456}
9457
9458Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v16f16_r(MVT RetVT, Register Op0) {
9459 if (RetVT.SimpleTy != MVT::v16i8)
9460 return Register();
9461 if ((Subtarget->hasAVX10_2())) {
9462 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9463 }
9464 return Register();
9465}
9466
9467Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v32f16_r(MVT RetVT, Register Op0) {
9468 if (RetVT.SimpleTy != MVT::v32i8)
9469 return Register();
9470 if ((Subtarget->hasAVX10_2())) {
9471 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZrr, RC: &X86::VR256XRegClass, Op0);
9472 }
9473 return Register();
9474}
9475
9476Register fastEmit_X86ISD_VCVTPH2HF8S_r(MVT VT, MVT RetVT, Register Op0) {
9477 switch (VT.SimpleTy) {
9478 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v8f16_r(RetVT, Op0);
9479 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v16f16_r(RetVT, Op0);
9480 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v32f16_r(RetVT, Op0);
9481 default: return Register();
9482 }
9483}
9484
9485// FastEmit functions for X86ISD::VFPEXT.
9486
9487Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Register Op0) {
9488 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9489 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
9490 }
9491 return Register();
9492}
9493
9494Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Register Op0) {
9495 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9496 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
9497 }
9498 return Register();
9499}
9500
9501Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Register Op0) {
9502 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9503 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
9504 }
9505 return Register();
9506}
9507
9508Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(MVT RetVT, Register Op0) {
9509switch (RetVT.SimpleTy) {
9510 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
9511 case MVT::v2f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
9512 case MVT::v4f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
9513 default: return Register();
9514}
9515}
9516
9517Register fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, Register Op0) {
9518 if (RetVT.SimpleTy != MVT::v2f64)
9519 return Register();
9520 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9521 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
9522 }
9523 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9524 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
9525 }
9526 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9527 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
9528 }
9529 return Register();
9530}
9531
9532Register fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, Register Op0) {
9533 switch (VT.SimpleTy) {
9534 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(RetVT, Op0);
9535 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0);
9536 default: return Register();
9537 }
9538}
9539
9540// FastEmit functions for X86ISD::VFPEXT_SAE.
9541
9542Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
9543 if (RetVT.SimpleTy != MVT::v8f64)
9544 return Register();
9545 if ((Subtarget->hasFP16())) {
9546 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrrb, RC: &X86::VR512RegClass, Op0);
9547 }
9548 return Register();
9549}
9550
9551Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
9552 if (RetVT.SimpleTy != MVT::v16f32)
9553 return Register();
9554 if ((Subtarget->hasFP16())) {
9555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrrb, RC: &X86::VR512RegClass, Op0);
9556 }
9557 return Register();
9558}
9559
9560Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
9561 if (RetVT.SimpleTy != MVT::v8f64)
9562 return Register();
9563 if ((Subtarget->hasAVX512())) {
9564 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrrb, RC: &X86::VR512RegClass, Op0);
9565 }
9566 return Register();
9567}
9568
9569Register fastEmit_X86ISD_VFPEXT_SAE_r(MVT VT, MVT RetVT, Register Op0) {
9570 switch (VT.SimpleTy) {
9571 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(RetVT, Op0);
9572 case MVT::v16f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(RetVT, Op0);
9573 case MVT::v8f32: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(RetVT, Op0);
9574 default: return Register();
9575 }
9576}
9577
9578// FastEmit functions for X86ISD::VFPROUND.
9579
9580Register fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
9581 if (RetVT.SimpleTy != MVT::v8f16)
9582 return Register();
9583 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9584 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
9585 }
9586 return Register();
9587}
9588
9589Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Register Op0) {
9590 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9591 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
9592 }
9593 return Register();
9594}
9595
9596Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Register Op0) {
9597 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9598 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
9599 }
9600 if ((Subtarget->hasAVXNECONVERT())) {
9601 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Yrr, RC: &X86::VR128RegClass, Op0);
9602 }
9603 return Register();
9604}
9605
9606Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(MVT RetVT, Register Op0) {
9607switch (RetVT.SimpleTy) {
9608 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Op0);
9609 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Op0);
9610 default: return Register();
9611}
9612}
9613
9614Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Register Op0) {
9615 if ((Subtarget->hasFP16())) {
9616 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
9617 }
9618 return Register();
9619}
9620
9621Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Register Op0) {
9622 if ((Subtarget->hasBF16())) {
9623 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
9624 }
9625 return Register();
9626}
9627
9628Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(MVT RetVT, Register Op0) {
9629switch (RetVT.SimpleTy) {
9630 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Op0);
9631 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Op0);
9632 default: return Register();
9633}
9634}
9635
9636Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Register Op0) {
9637 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9638 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
9639 }
9640 return Register();
9641}
9642
9643Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Register Op0) {
9644 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9645 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
9646 }
9647 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9648 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
9649 }
9650 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9651 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
9652 }
9653 return Register();
9654}
9655
9656Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
9657switch (RetVT.SimpleTy) {
9658 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
9659 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
9660 default: return Register();
9661}
9662}
9663
9664Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Register Op0) {
9665 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9666 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
9667 }
9668 return Register();
9669}
9670
9671Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Register Op0) {
9672 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9673 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
9674 }
9675 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9676 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
9677 }
9678 return Register();
9679}
9680
9681Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(MVT RetVT, Register Op0) {
9682switch (RetVT.SimpleTy) {
9683 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
9684 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
9685 default: return Register();
9686}
9687}
9688
9689Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Register Op0) {
9690 if ((Subtarget->hasFP16())) {
9691 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
9692 }
9693 return Register();
9694}
9695
9696Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Register Op0) {
9697 if ((Subtarget->hasAVX512())) {
9698 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
9699 }
9700 return Register();
9701}
9702
9703Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(MVT RetVT, Register Op0) {
9704switch (RetVT.SimpleTy) {
9705 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
9706 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
9707 default: return Register();
9708}
9709}
9710
9711Register fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, Register Op0) {
9712 switch (VT.SimpleTy) {
9713 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(RetVT, Op0);
9714 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(RetVT, Op0);
9715 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(RetVT, Op0);
9716 case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0);
9717 case MVT::v4f64: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(RetVT, Op0);
9718 case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(RetVT, Op0);
9719 default: return Register();
9720 }
9721}
9722
9723// FastEmit functions for X86ISD::VTRUNC.
9724
9725Register fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, Register Op0) {
9726 if (RetVT.SimpleTy != MVT::v16i8)
9727 return Register();
9728 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9729 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ128rr, RC: &X86::VR128XRegClass, Op0);
9730 }
9731 return Register();
9732}
9733
9734Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Register Op0) {
9735 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9736 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ128rr, RC: &X86::VR128XRegClass, Op0);
9737 }
9738 return Register();
9739}
9740
9741Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Register Op0) {
9742 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9743 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ128rr, RC: &X86::VR128XRegClass, Op0);
9744 }
9745 return Register();
9746}
9747
9748Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, Register Op0) {
9749switch (RetVT.SimpleTy) {
9750 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0);
9751 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0);
9752 default: return Register();
9753}
9754}
9755
9756Register fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, Register Op0) {
9757 if (RetVT.SimpleTy != MVT::v16i8)
9758 return Register();
9759 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9760 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ256rr, RC: &X86::VR128XRegClass, Op0);
9761 }
9762 return Register();
9763}
9764
9765Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Register Op0) {
9766 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9767 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ128rr, RC: &X86::VR128XRegClass, Op0);
9768 }
9769 return Register();
9770}
9771
9772Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Register Op0) {
9773 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9774 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ128rr, RC: &X86::VR128XRegClass, Op0);
9775 }
9776 return Register();
9777}
9778
9779Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Register Op0) {
9780 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9781 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ128rr, RC: &X86::VR128XRegClass, Op0);
9782 }
9783 return Register();
9784}
9785
9786Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, Register Op0) {
9787switch (RetVT.SimpleTy) {
9788 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0);
9789 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0);
9790 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0);
9791 default: return Register();
9792}
9793}
9794
9795Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Register Op0) {
9796 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9797 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ256rr, RC: &X86::VR128XRegClass, Op0);
9798 }
9799 return Register();
9800}
9801
9802Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Register Op0) {
9803 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9804 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ256rr, RC: &X86::VR128XRegClass, Op0);
9805 }
9806 return Register();
9807}
9808
9809Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, Register Op0) {
9810switch (RetVT.SimpleTy) {
9811 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0);
9812 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0);
9813 default: return Register();
9814}
9815}
9816
9817Register fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, Register Op0) {
9818 if (RetVT.SimpleTy != MVT::v16i8)
9819 return Register();
9820 if ((Subtarget->hasAVX512())) {
9821 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZrr, RC: &X86::VR128XRegClass, Op0);
9822 }
9823 return Register();
9824}
9825
9826Register fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
9827 switch (VT.SimpleTy) {
9828 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0);
9829 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0);
9830 case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0);
9831 case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0);
9832 case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0);
9833 case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0);
9834 default: return Register();
9835 }
9836}
9837
9838// FastEmit functions for X86ISD::VTRUNCS.
9839
9840Register fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, Register Op0) {
9841 if (RetVT.SimpleTy != MVT::v16i8)
9842 return Register();
9843 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9844 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
9845 }
9846 return Register();
9847}
9848
9849Register fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, Register Op0) {
9850 if (RetVT.SimpleTy != MVT::v16i8)
9851 return Register();
9852 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9853 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
9854 }
9855 return Register();
9856}
9857
9858Register fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, Register Op0) {
9859 if (RetVT.SimpleTy != MVT::v32i8)
9860 return Register();
9861 if ((Subtarget->hasBWI())) {
9862 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZrr, RC: &X86::VR256XRegClass, Op0);
9863 }
9864 return Register();
9865}
9866
9867Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Register Op0) {
9868 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9869 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
9870 }
9871 return Register();
9872}
9873
9874Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Register Op0) {
9875 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9876 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
9877 }
9878 return Register();
9879}
9880
9881Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, Register Op0) {
9882switch (RetVT.SimpleTy) {
9883 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0);
9884 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0);
9885 default: return Register();
9886}
9887}
9888
9889Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Register Op0) {
9890 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9891 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
9892 }
9893 return Register();
9894}
9895
9896Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Register Op0) {
9897 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9898 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
9899 }
9900 return Register();
9901}
9902
9903Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, Register Op0) {
9904switch (RetVT.SimpleTy) {
9905 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0);
9906 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0);
9907 default: return Register();
9908}
9909}
9910
9911Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Register Op0) {
9912 if ((Subtarget->hasAVX512())) {
9913 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZrr, RC: &X86::VR128XRegClass, Op0);
9914 }
9915 return Register();
9916}
9917
9918Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Register Op0) {
9919 if ((Subtarget->hasAVX512())) {
9920 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZrr, RC: &X86::VR256XRegClass, Op0);
9921 }
9922 return Register();
9923}
9924
9925Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, Register Op0) {
9926switch (RetVT.SimpleTy) {
9927 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0);
9928 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0);
9929 default: return Register();
9930}
9931}
9932
9933Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Register Op0) {
9934 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9935 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
9936 }
9937 return Register();
9938}
9939
9940Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Register Op0) {
9941 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9942 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
9943 }
9944 return Register();
9945}
9946
9947Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Register Op0) {
9948 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9949 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
9950 }
9951 return Register();
9952}
9953
9954Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, Register Op0) {
9955switch (RetVT.SimpleTy) {
9956 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0);
9957 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0);
9958 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0);
9959 default: return Register();
9960}
9961}
9962
9963Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Register Op0) {
9964 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9965 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
9966 }
9967 return Register();
9968}
9969
9970Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Register Op0) {
9971 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9972 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
9973 }
9974 return Register();
9975}
9976
9977Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Register Op0) {
9978 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9979 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
9980 }
9981 return Register();
9982}
9983
9984Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, Register Op0) {
9985switch (RetVT.SimpleTy) {
9986 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0);
9987 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0);
9988 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0);
9989 default: return Register();
9990}
9991}
9992
9993Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Register Op0) {
9994 if ((Subtarget->hasAVX512())) {
9995 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZrr, RC: &X86::VR128XRegClass, Op0);
9996 }
9997 return Register();
9998}
9999
10000Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Register Op0) {
10001 if ((Subtarget->hasAVX512())) {
10002 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZrr, RC: &X86::VR128XRegClass, Op0);
10003 }
10004 return Register();
10005}
10006
10007Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Register Op0) {
10008 if ((Subtarget->hasAVX512())) {
10009 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZrr, RC: &X86::VR256XRegClass, Op0);
10010 }
10011 return Register();
10012}
10013
10014Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, Register Op0) {
10015switch (RetVT.SimpleTy) {
10016 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0);
10017 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0);
10018 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0);
10019 default: return Register();
10020}
10021}
10022
10023Register fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, Register Op0) {
10024 switch (VT.SimpleTy) {
10025 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0);
10026 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0);
10027 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0);
10028 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0);
10029 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0);
10030 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0);
10031 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0);
10032 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0);
10033 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0);
10034 default: return Register();
10035 }
10036}
10037
10038// FastEmit functions for X86ISD::VTRUNCUS.
10039
10040Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, Register Op0) {
10041 if (RetVT.SimpleTy != MVT::v16i8)
10042 return Register();
10043 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10044 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
10045 }
10046 return Register();
10047}
10048
10049Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, Register Op0) {
10050 if (RetVT.SimpleTy != MVT::v16i8)
10051 return Register();
10052 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10053 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
10054 }
10055 return Register();
10056}
10057
10058Register fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, Register Op0) {
10059 if (RetVT.SimpleTy != MVT::v32i8)
10060 return Register();
10061 if ((Subtarget->hasBWI())) {
10062 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZrr, RC: &X86::VR256XRegClass, Op0);
10063 }
10064 return Register();
10065}
10066
10067Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Register Op0) {
10068 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10069 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
10070 }
10071 return Register();
10072}
10073
10074Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Register Op0) {
10075 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10076 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
10077 }
10078 return Register();
10079}
10080
10081Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, Register Op0) {
10082switch (RetVT.SimpleTy) {
10083 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0);
10084 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0);
10085 default: return Register();
10086}
10087}
10088
10089Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Register Op0) {
10090 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10091 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
10092 }
10093 return Register();
10094}
10095
10096Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Register Op0) {
10097 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10098 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
10099 }
10100 return Register();
10101}
10102
10103Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, Register Op0) {
10104switch (RetVT.SimpleTy) {
10105 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0);
10106 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0);
10107 default: return Register();
10108}
10109}
10110
10111Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Register Op0) {
10112 if ((Subtarget->hasAVX512())) {
10113 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZrr, RC: &X86::VR128XRegClass, Op0);
10114 }
10115 return Register();
10116}
10117
10118Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Register Op0) {
10119 if ((Subtarget->hasAVX512())) {
10120 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZrr, RC: &X86::VR256XRegClass, Op0);
10121 }
10122 return Register();
10123}
10124
10125Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, Register Op0) {
10126switch (RetVT.SimpleTy) {
10127 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0);
10128 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0);
10129 default: return Register();
10130}
10131}
10132
10133Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Register Op0) {
10134 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10135 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
10136 }
10137 return Register();
10138}
10139
10140Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Register Op0) {
10141 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10142 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
10143 }
10144 return Register();
10145}
10146
10147Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Register Op0) {
10148 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10149 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
10150 }
10151 return Register();
10152}
10153
10154Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, Register Op0) {
10155switch (RetVT.SimpleTy) {
10156 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0);
10157 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0);
10158 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0);
10159 default: return Register();
10160}
10161}
10162
10163Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Register Op0) {
10164 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10165 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
10166 }
10167 return Register();
10168}
10169
10170Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Register Op0) {
10171 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10172 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
10173 }
10174 return Register();
10175}
10176
10177Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Register Op0) {
10178 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10179 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
10180 }
10181 return Register();
10182}
10183
10184Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, Register Op0) {
10185switch (RetVT.SimpleTy) {
10186 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0);
10187 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0);
10188 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0);
10189 default: return Register();
10190}
10191}
10192
10193Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Register Op0) {
10194 if ((Subtarget->hasAVX512())) {
10195 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZrr, RC: &X86::VR128XRegClass, Op0);
10196 }
10197 return Register();
10198}
10199
10200Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Register Op0) {
10201 if ((Subtarget->hasAVX512())) {
10202 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZrr, RC: &X86::VR128XRegClass, Op0);
10203 }
10204 return Register();
10205}
10206
10207Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Register Op0) {
10208 if ((Subtarget->hasAVX512())) {
10209 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZrr, RC: &X86::VR256XRegClass, Op0);
10210 }
10211 return Register();
10212}
10213
10214Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, Register Op0) {
10215switch (RetVT.SimpleTy) {
10216 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0);
10217 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0);
10218 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0);
10219 default: return Register();
10220}
10221}
10222
10223Register fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, Register Op0) {
10224 switch (VT.SimpleTy) {
10225 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0);
10226 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0);
10227 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0);
10228 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0);
10229 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0);
10230 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0);
10231 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0);
10232 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0);
10233 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0);
10234 default: return Register();
10235 }
10236}
10237
10238// FastEmit functions for X86ISD::VZEXT_MOVL.
10239
10240Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v8i16_r(MVT RetVT, Register Op0) {
10241 if (RetVT.SimpleTy != MVT::v8i16)
10242 return Register();
10243 if ((Subtarget->hasAVX10_2())) {
10244 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPWILo2PWIZrr, RC: &X86::VR128XRegClass, Op0);
10245 }
10246 return Register();
10247}
10248
10249Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v4i32_r(MVT RetVT, Register Op0) {
10250 if (RetVT.SimpleTy != MVT::v4i32)
10251 return Register();
10252 if ((Subtarget->hasAVX10_2())) {
10253 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPDILo2PDIZrr, RC: &X86::VR128XRegClass, Op0);
10254 }
10255 return Register();
10256}
10257
10258Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, Register Op0) {
10259 if (RetVT.SimpleTy != MVT::v2i64)
10260 return Register();
10261 if ((Subtarget->hasAVX512())) {
10262 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
10263 }
10264 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10265 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10266 }
10267 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10268 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10269 }
10270 return Register();
10271}
10272
10273Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, Register Op0) {
10274 if (RetVT.SimpleTy != MVT::v2f64)
10275 return Register();
10276 if ((Subtarget->hasAVX512())) {
10277 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
10278 }
10279 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10280 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10281 }
10282 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10283 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10284 }
10285 return Register();
10286}
10287
10288Register fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, Register Op0) {
10289 switch (VT.SimpleTy) {
10290 case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v8i16_r(RetVT, Op0);
10291 case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v4i32_r(RetVT, Op0);
10292 case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0);
10293 case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0);
10294 default: return Register();
10295 }
10296}
10297
10298// Top-level FastEmit function.
10299
10300Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
10301 switch (Opcode) {
10302 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
10303 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
10304 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
10305 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
10306 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
10307 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
10308 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
10309 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
10310 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
10311 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
10312 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
10313 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
10314 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
10315 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
10316 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
10317 case ISD::LLRINT: return fastEmit_ISD_LLRINT_r(VT, RetVT, Op0);
10318 case ISD::LRINT: return fastEmit_ISD_LRINT_r(VT, RetVT, Op0);
10319 case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0);
10320 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
10321 case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
10322 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
10323 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
10324 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
10325 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
10326 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
10327 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
10328 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
10329 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
10330 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
10331 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
10332 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
10333 case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
10334 case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0);
10335 case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0);
10336 case X86ISD::CVTNEPS2BF16: return fastEmit_X86ISD_CVTNEPS2BF16_r(VT, RetVT, Op0);
10337 case X86ISD::CVTP2IBS: return fastEmit_X86ISD_CVTP2IBS_r(VT, RetVT, Op0);
10338 case X86ISD::CVTP2IUBS: return fastEmit_X86ISD_CVTP2IUBS_r(VT, RetVT, Op0);
10339 case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0);
10340 case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0);
10341 case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0);
10342 case X86ISD::CVTPH2PS_SAE: return fastEmit_X86ISD_CVTPH2PS_SAE_r(VT, RetVT, Op0);
10343 case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0);
10344 case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0);
10345 case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0);
10346 case X86ISD::CVTTP2IBS: return fastEmit_X86ISD_CVTTP2IBS_r(VT, RetVT, Op0);
10347 case X86ISD::CVTTP2IBS_SAE: return fastEmit_X86ISD_CVTTP2IBS_SAE_r(VT, RetVT, Op0);
10348 case X86ISD::CVTTP2IUBS: return fastEmit_X86ISD_CVTTP2IUBS_r(VT, RetVT, Op0);
10349 case X86ISD::CVTTP2IUBS_SAE: return fastEmit_X86ISD_CVTTP2IUBS_SAE_r(VT, RetVT, Op0);
10350 case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0);
10351 case X86ISD::CVTTP2SIS: return fastEmit_X86ISD_CVTTP2SIS_r(VT, RetVT, Op0);
10352 case X86ISD::CVTTP2SIS_SAE: return fastEmit_X86ISD_CVTTP2SIS_SAE_r(VT, RetVT, Op0);
10353 case X86ISD::CVTTP2SI_SAE: return fastEmit_X86ISD_CVTTP2SI_SAE_r(VT, RetVT, Op0);
10354 case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0);
10355 case X86ISD::CVTTP2UIS: return fastEmit_X86ISD_CVTTP2UIS_r(VT, RetVT, Op0);
10356 case X86ISD::CVTTP2UIS_SAE: return fastEmit_X86ISD_CVTTP2UIS_SAE_r(VT, RetVT, Op0);
10357 case X86ISD::CVTTP2UI_SAE: return fastEmit_X86ISD_CVTTP2UI_SAE_r(VT, RetVT, Op0);
10358 case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0);
10359 case X86ISD::CVTTS2SIS: return fastEmit_X86ISD_CVTTS2SIS_r(VT, RetVT, Op0);
10360 case X86ISD::CVTTS2SIS_SAE: return fastEmit_X86ISD_CVTTS2SIS_SAE_r(VT, RetVT, Op0);
10361 case X86ISD::CVTTS2SI_SAE: return fastEmit_X86ISD_CVTTS2SI_SAE_r(VT, RetVT, Op0);
10362 case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0);
10363 case X86ISD::CVTTS2UIS: return fastEmit_X86ISD_CVTTS2UIS_r(VT, RetVT, Op0);
10364 case X86ISD::CVTTS2UIS_SAE: return fastEmit_X86ISD_CVTTS2UIS_SAE_r(VT, RetVT, Op0);
10365 case X86ISD::CVTTS2UI_SAE: return fastEmit_X86ISD_CVTTS2UI_SAE_r(VT, RetVT, Op0);
10366 case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0);
10367 case X86ISD::DYN_ALLOCA: return fastEmit_X86ISD_DYN_ALLOCA_r(VT, RetVT, Op0);
10368 case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0);
10369 case X86ISD::FGETEXP: return fastEmit_X86ISD_FGETEXP_r(VT, RetVT, Op0);
10370 case X86ISD::FGETEXP_SAE: return fastEmit_X86ISD_FGETEXP_SAE_r(VT, RetVT, Op0);
10371 case X86ISD::FP_TO_SINT_SAT: return fastEmit_X86ISD_FP_TO_SINT_SAT_r(VT, RetVT, Op0);
10372 case X86ISD::FP_TO_UINT_SAT: return fastEmit_X86ISD_FP_TO_UINT_SAT_r(VT, RetVT, Op0);
10373 case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0);
10374 case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0);
10375 case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0);
10376 case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0);
10377 case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0);
10378 case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0);
10379 case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0);
10380 case X86ISD::MOVQ2DQ: return fastEmit_X86ISD_MOVQ2DQ_r(VT, RetVT, Op0);
10381 case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0);
10382 case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0);
10383 case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0);
10384 case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0);
10385 case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0);
10386 case X86ISD::PROBED_ALLOCA: return fastEmit_X86ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
10387 case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0);
10388 case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0);
10389 case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0);
10390 case X86ISD::STRICT_CVTPH2PS: return fastEmit_X86ISD_STRICT_CVTPH2PS_r(VT, RetVT, Op0);
10391 case X86ISD::STRICT_CVTSI2P: return fastEmit_X86ISD_STRICT_CVTSI2P_r(VT, RetVT, Op0);
10392 case X86ISD::STRICT_CVTTP2SI: return fastEmit_X86ISD_STRICT_CVTTP2SI_r(VT, RetVT, Op0);
10393 case X86ISD::STRICT_CVTTP2UI: return fastEmit_X86ISD_STRICT_CVTTP2UI_r(VT, RetVT, Op0);
10394 case X86ISD::STRICT_CVTUI2P: return fastEmit_X86ISD_STRICT_CVTUI2P_r(VT, RetVT, Op0);
10395 case X86ISD::STRICT_VFPEXT: return fastEmit_X86ISD_STRICT_VFPEXT_r(VT, RetVT, Op0);
10396 case X86ISD::STRICT_VFPROUND: return fastEmit_X86ISD_STRICT_VFPROUND_r(VT, RetVT, Op0);
10397 case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0);
10398 case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0);
10399 case X86ISD::VCVTHF82PH: return fastEmit_X86ISD_VCVTHF82PH_r(VT, RetVT, Op0);
10400 case X86ISD::VCVTPH2BF8: return fastEmit_X86ISD_VCVTPH2BF8_r(VT, RetVT, Op0);
10401 case X86ISD::VCVTPH2BF8S: return fastEmit_X86ISD_VCVTPH2BF8S_r(VT, RetVT, Op0);
10402 case X86ISD::VCVTPH2HF8: return fastEmit_X86ISD_VCVTPH2HF8_r(VT, RetVT, Op0);
10403 case X86ISD::VCVTPH2HF8S: return fastEmit_X86ISD_VCVTPH2HF8S_r(VT, RetVT, Op0);
10404 case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0);
10405 case X86ISD::VFPEXT_SAE: return fastEmit_X86ISD_VFPEXT_SAE_r(VT, RetVT, Op0);
10406 case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0);
10407 case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0);
10408 case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0);
10409 case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0);
10410 case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0);
10411 default: return Register();
10412 }
10413}
10414
10415// FastEmit functions for ISD::ADD.
10416
10417Register fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
10418 if (RetVT.SimpleTy != MVT::i8)
10419 return Register();
10420 if ((Subtarget->hasNDD())) {
10421 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10422 }
10423 if ((!Subtarget->hasNDD())) {
10424 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr, RC: &X86::GR8RegClass, Op0, Op1);
10425 }
10426 return Register();
10427}
10428
10429Register fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
10430 if (RetVT.SimpleTy != MVT::i16)
10431 return Register();
10432 if ((Subtarget->hasNDD())) {
10433 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10434 }
10435 if ((!Subtarget->hasNDD())) {
10436 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr, RC: &X86::GR16RegClass, Op0, Op1);
10437 }
10438 return Register();
10439}
10440
10441Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10442 if (RetVT.SimpleTy != MVT::i32)
10443 return Register();
10444 if ((Subtarget->hasNDD())) {
10445 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10446 }
10447 if ((!Subtarget->hasNDD())) {
10448 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr, RC: &X86::GR32RegClass, Op0, Op1);
10449 }
10450 return Register();
10451}
10452
10453Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10454 if (RetVT.SimpleTy != MVT::i64)
10455 return Register();
10456 if ((Subtarget->hasNDD())) {
10457 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10458 }
10459 if ((!Subtarget->hasNDD())) {
10460 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr, RC: &X86::GR64RegClass, Op0, Op1);
10461 }
10462 return Register();
10463}
10464
10465Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10466 if (RetVT.SimpleTy != MVT::v16i8)
10467 return Register();
10468 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10469 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10470 }
10471 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10472 return fastEmitInst_rr(MachineInstOpcode: X86::PADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
10473 }
10474 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10475 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
10476 }
10477 return Register();
10478}
10479
10480Register fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10481 if (RetVT.SimpleTy != MVT::v32i8)
10482 return Register();
10483 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10484 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10485 }
10486 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10487 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10488 }
10489 return Register();
10490}
10491
10492Register fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10493 if (RetVT.SimpleTy != MVT::v64i8)
10494 return Register();
10495 if ((Subtarget->hasBWI())) {
10496 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10497 }
10498 return Register();
10499}
10500
10501Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10502 if (RetVT.SimpleTy != MVT::v8i16)
10503 return Register();
10504 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10505 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10506 }
10507 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10508 return fastEmitInst_rr(MachineInstOpcode: X86::PADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
10509 }
10510 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10511 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
10512 }
10513 return Register();
10514}
10515
10516Register fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10517 if (RetVT.SimpleTy != MVT::v16i16)
10518 return Register();
10519 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10520 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10521 }
10522 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10523 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10524 }
10525 return Register();
10526}
10527
10528Register fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10529 if (RetVT.SimpleTy != MVT::v32i16)
10530 return Register();
10531 if ((Subtarget->hasBWI())) {
10532 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10533 }
10534 return Register();
10535}
10536
10537Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10538 if (RetVT.SimpleTy != MVT::v4i32)
10539 return Register();
10540 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10541 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10542 }
10543 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10544 return fastEmitInst_rr(MachineInstOpcode: X86::PADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
10545 }
10546 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10547 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
10548 }
10549 return Register();
10550}
10551
10552Register fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
10553 if (RetVT.SimpleTy != MVT::v8i32)
10554 return Register();
10555 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10556 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10557 }
10558 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10559 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10560 }
10561 return Register();
10562}
10563
10564Register fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
10565 if (RetVT.SimpleTy != MVT::v16i32)
10566 return Register();
10567 if ((Subtarget->hasAVX512())) {
10568 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10569 }
10570 return Register();
10571}
10572
10573Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10574 if (RetVT.SimpleTy != MVT::v2i64)
10575 return Register();
10576 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10577 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10578 }
10579 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10580 return fastEmitInst_rr(MachineInstOpcode: X86::PADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
10581 }
10582 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10583 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
10584 }
10585 return Register();
10586}
10587
10588Register fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
10589 if (RetVT.SimpleTy != MVT::v4i64)
10590 return Register();
10591 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10592 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10593 }
10594 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10595 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
10596 }
10597 return Register();
10598}
10599
10600Register fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
10601 if (RetVT.SimpleTy != MVT::v8i64)
10602 return Register();
10603 if ((Subtarget->hasAVX512())) {
10604 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10605 }
10606 return Register();
10607}
10608
10609Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10610 switch (VT.SimpleTy) {
10611 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op1);
10612 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op1);
10613 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10614 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10615 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10616 case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op1);
10617 case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op1);
10618 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10619 case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op1);
10620 case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op1);
10621 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10622 case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op1);
10623 case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op1);
10624 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10625 case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op1);
10626 case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op1);
10627 default: return Register();
10628 }
10629}
10630
10631// FastEmit functions for ISD::AND.
10632
10633Register fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
10634 if (RetVT.SimpleTy != MVT::i8)
10635 return Register();
10636 if ((Subtarget->hasNDD())) {
10637 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10638 }
10639 if ((!Subtarget->hasNDD())) {
10640 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr, RC: &X86::GR8RegClass, Op0, Op1);
10641 }
10642 return Register();
10643}
10644
10645Register fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
10646 if (RetVT.SimpleTy != MVT::i16)
10647 return Register();
10648 if ((Subtarget->hasNDD())) {
10649 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10650 }
10651 if ((!Subtarget->hasNDD())) {
10652 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr, RC: &X86::GR16RegClass, Op0, Op1);
10653 }
10654 return Register();
10655}
10656
10657Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10658 if (RetVT.SimpleTy != MVT::i32)
10659 return Register();
10660 if ((Subtarget->hasNDD())) {
10661 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10662 }
10663 if ((!Subtarget->hasNDD())) {
10664 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr, RC: &X86::GR32RegClass, Op0, Op1);
10665 }
10666 return Register();
10667}
10668
10669Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10670 if (RetVT.SimpleTy != MVT::i64)
10671 return Register();
10672 if ((Subtarget->hasNDD())) {
10673 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10674 }
10675 if ((!Subtarget->hasNDD())) {
10676 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr, RC: &X86::GR64RegClass, Op0, Op1);
10677 }
10678 return Register();
10679}
10680
10681Register fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10682 if (RetVT.SimpleTy != MVT::v8i1)
10683 return Register();
10684 if ((Subtarget->hasDQI())) {
10685 return fastEmitInst_rr(MachineInstOpcode: X86::KANDBkk, RC: &X86::VK8RegClass, Op0, Op1);
10686 }
10687 return Register();
10688}
10689
10690Register fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
10691 if (RetVT.SimpleTy != MVT::v16i1)
10692 return Register();
10693 if ((Subtarget->hasAVX512())) {
10694 return fastEmitInst_rr(MachineInstOpcode: X86::KANDWkk, RC: &X86::VK16RegClass, Op0, Op1);
10695 }
10696 return Register();
10697}
10698
10699Register fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
10700 if (RetVT.SimpleTy != MVT::v32i1)
10701 return Register();
10702 if ((Subtarget->hasBWI())) {
10703 return fastEmitInst_rr(MachineInstOpcode: X86::KANDDkk, RC: &X86::VK32RegClass, Op0, Op1);
10704 }
10705 return Register();
10706}
10707
10708Register fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
10709 if (RetVT.SimpleTy != MVT::v64i1)
10710 return Register();
10711 if ((Subtarget->hasBWI())) {
10712 return fastEmitInst_rr(MachineInstOpcode: X86::KANDQkk, RC: &X86::VK64RegClass, Op0, Op1);
10713 }
10714 return Register();
10715}
10716
10717Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10718 if (RetVT.SimpleTy != MVT::v16i8)
10719 return Register();
10720 if ((Subtarget->hasVLX())) {
10721 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10722 }
10723 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10724 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10725 }
10726 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10727 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10728 }
10729 return Register();
10730}
10731
10732Register fastEmit_ISD_AND_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10733 if (RetVT.SimpleTy != MVT::v32i8)
10734 return Register();
10735 if ((Subtarget->hasVLX())) {
10736 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10737 }
10738 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10739 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10740 }
10741 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10742 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10743 }
10744 return Register();
10745}
10746
10747Register fastEmit_ISD_AND_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10748 if (RetVT.SimpleTy != MVT::v64i8)
10749 return Register();
10750 if ((Subtarget->hasAVX512())) {
10751 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10752 }
10753 return Register();
10754}
10755
10756Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10757 if (RetVT.SimpleTy != MVT::v8i16)
10758 return Register();
10759 if ((Subtarget->hasVLX())) {
10760 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10761 }
10762 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10763 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10764 }
10765 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10766 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10767 }
10768 return Register();
10769}
10770
10771Register fastEmit_ISD_AND_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10772 if (RetVT.SimpleTy != MVT::v16i16)
10773 return Register();
10774 if ((Subtarget->hasVLX())) {
10775 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10776 }
10777 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10778 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10779 }
10780 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10781 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10782 }
10783 return Register();
10784}
10785
10786Register fastEmit_ISD_AND_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10787 if (RetVT.SimpleTy != MVT::v32i16)
10788 return Register();
10789 if ((Subtarget->hasAVX512())) {
10790 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10791 }
10792 return Register();
10793}
10794
10795Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10796 if (RetVT.SimpleTy != MVT::v4i32)
10797 return Register();
10798 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10799 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10800 }
10801 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10802 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10803 }
10804 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10805 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10806 }
10807 return Register();
10808}
10809
10810Register fastEmit_ISD_AND_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
10811 if (RetVT.SimpleTy != MVT::v8i32)
10812 return Register();
10813 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10814 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10815 }
10816 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10817 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10818 }
10819 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10820 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10821 }
10822 return Register();
10823}
10824
10825Register fastEmit_ISD_AND_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
10826 if (RetVT.SimpleTy != MVT::v16i32)
10827 return Register();
10828 if ((Subtarget->hasAVX512())) {
10829 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10830 }
10831 return Register();
10832}
10833
10834Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10835 if (RetVT.SimpleTy != MVT::v2i64)
10836 return Register();
10837 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10838 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10839 }
10840 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10841 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10842 }
10843 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10844 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10845 }
10846 return Register();
10847}
10848
10849Register fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
10850 if (RetVT.SimpleTy != MVT::v4i64)
10851 return Register();
10852 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10853 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10854 }
10855 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10856 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10857 }
10858 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10859 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10860 }
10861 return Register();
10862}
10863
10864Register fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
10865 if (RetVT.SimpleTy != MVT::v8i64)
10866 return Register();
10867 if ((Subtarget->hasAVX512())) {
10868 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10869 }
10870 return Register();
10871}
10872
10873Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10874 switch (VT.SimpleTy) {
10875 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op1);
10876 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op1);
10877 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10878 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10879 case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op1);
10880 case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op1);
10881 case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op1);
10882 case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op1);
10883 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10884 case MVT::v32i8: return fastEmit_ISD_AND_MVT_v32i8_rr(RetVT, Op0, Op1);
10885 case MVT::v64i8: return fastEmit_ISD_AND_MVT_v64i8_rr(RetVT, Op0, Op1);
10886 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10887 case MVT::v16i16: return fastEmit_ISD_AND_MVT_v16i16_rr(RetVT, Op0, Op1);
10888 case MVT::v32i16: return fastEmit_ISD_AND_MVT_v32i16_rr(RetVT, Op0, Op1);
10889 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10890 case MVT::v8i32: return fastEmit_ISD_AND_MVT_v8i32_rr(RetVT, Op0, Op1);
10891 case MVT::v16i32: return fastEmit_ISD_AND_MVT_v16i32_rr(RetVT, Op0, Op1);
10892 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10893 case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op1);
10894 case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op1);
10895 default: return Register();
10896 }
10897}
10898
10899// FastEmit functions for ISD::AVGCEILU.
10900
10901Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10902 if (RetVT.SimpleTy != MVT::v16i8)
10903 return Register();
10904 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10905 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10906 }
10907 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10908 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
10909 }
10910 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10911 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
10912 }
10913 return Register();
10914}
10915
10916Register fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10917 if (RetVT.SimpleTy != MVT::v32i8)
10918 return Register();
10919 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10920 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10921 }
10922 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10923 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10924 }
10925 return Register();
10926}
10927
10928Register fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10929 if (RetVT.SimpleTy != MVT::v64i8)
10930 return Register();
10931 if ((Subtarget->hasBWI())) {
10932 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10933 }
10934 return Register();
10935}
10936
10937Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10938 if (RetVT.SimpleTy != MVT::v8i16)
10939 return Register();
10940 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10941 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10942 }
10943 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10944 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
10945 }
10946 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10947 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
10948 }
10949 return Register();
10950}
10951
10952Register fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10953 if (RetVT.SimpleTy != MVT::v16i16)
10954 return Register();
10955 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10956 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10957 }
10958 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10959 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10960 }
10961 return Register();
10962}
10963
10964Register fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10965 if (RetVT.SimpleTy != MVT::v32i16)
10966 return Register();
10967 if ((Subtarget->hasBWI())) {
10968 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10969 }
10970 return Register();
10971}
10972
10973Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10974 switch (VT.SimpleTy) {
10975 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10976 case MVT::v32i8: return fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(RetVT, Op0, Op1);
10977 case MVT::v64i8: return fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(RetVT, Op0, Op1);
10978 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10979 case MVT::v16i16: return fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(RetVT, Op0, Op1);
10980 case MVT::v32i16: return fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(RetVT, Op0, Op1);
10981 default: return Register();
10982 }
10983}
10984
10985// FastEmit functions for ISD::FADD.
10986
10987Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10988 if (RetVT.SimpleTy != MVT::f16)
10989 return Register();
10990 if ((Subtarget->hasFP16())) {
10991 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
10992 }
10993 return Register();
10994}
10995
10996Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10997 if (RetVT.SimpleTy != MVT::f32)
10998 return Register();
10999 if ((Subtarget->hasAVX512())) {
11000 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11001 }
11002 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11003 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11004 }
11005 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11006 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11007 }
11008 if ((!Subtarget->hasSSE1())) {
11009 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11010 }
11011 return Register();
11012}
11013
11014Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11015 if (RetVT.SimpleTy != MVT::f64)
11016 return Register();
11017 if ((Subtarget->hasAVX512())) {
11018 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11019 }
11020 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11021 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11022 }
11023 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11024 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11025 }
11026 if ((!Subtarget->hasSSE2())) {
11027 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11028 }
11029 return Register();
11030}
11031
11032Register fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11033 if (RetVT.SimpleTy != MVT::f80)
11034 return Register();
11035 if ((Subtarget->hasX87())) {
11036 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11037 }
11038 return Register();
11039}
11040
11041Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11042 if (RetVT.SimpleTy != MVT::v8f16)
11043 return Register();
11044 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11045 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11046 }
11047 return Register();
11048}
11049
11050Register fastEmit_ISD_FADD_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11051 if (RetVT.SimpleTy != MVT::v16f16)
11052 return Register();
11053 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11054 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11055 }
11056 return Register();
11057}
11058
11059Register fastEmit_ISD_FADD_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11060 if (RetVT.SimpleTy != MVT::v32f16)
11061 return Register();
11062 if ((Subtarget->hasFP16())) {
11063 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11064 }
11065 return Register();
11066}
11067
11068Register fastEmit_ISD_FADD_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11069 if (RetVT.SimpleTy != MVT::v8bf16)
11070 return Register();
11071 if ((Subtarget->hasAVX10_2())) {
11072 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11073 }
11074 return Register();
11075}
11076
11077Register fastEmit_ISD_FADD_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11078 if (RetVT.SimpleTy != MVT::v16bf16)
11079 return Register();
11080 if ((Subtarget->hasAVX10_2())) {
11081 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11082 }
11083 return Register();
11084}
11085
11086Register fastEmit_ISD_FADD_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11087 if (RetVT.SimpleTy != MVT::v32bf16)
11088 return Register();
11089 if ((Subtarget->hasAVX10_2())) {
11090 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11091 }
11092 return Register();
11093}
11094
11095Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11096 if (RetVT.SimpleTy != MVT::v4f32)
11097 return Register();
11098 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11099 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11100 }
11101 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11102 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11103 }
11104 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11105 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11106 }
11107 return Register();
11108}
11109
11110Register fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11111 if (RetVT.SimpleTy != MVT::v8f32)
11112 return Register();
11113 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11114 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11115 }
11116 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11117 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11118 }
11119 return Register();
11120}
11121
11122Register fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11123 if (RetVT.SimpleTy != MVT::v16f32)
11124 return Register();
11125 if ((Subtarget->hasAVX512())) {
11126 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11127 }
11128 return Register();
11129}
11130
11131Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11132 if (RetVT.SimpleTy != MVT::v2f64)
11133 return Register();
11134 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11135 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11136 }
11137 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11138 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11139 }
11140 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11141 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11142 }
11143 return Register();
11144}
11145
11146Register fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11147 if (RetVT.SimpleTy != MVT::v4f64)
11148 return Register();
11149 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11150 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11151 }
11152 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11153 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11154 }
11155 return Register();
11156}
11157
11158Register fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11159 if (RetVT.SimpleTy != MVT::v8f64)
11160 return Register();
11161 if ((Subtarget->hasAVX512())) {
11162 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11163 }
11164 return Register();
11165}
11166
11167Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11168 switch (VT.SimpleTy) {
11169 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11170 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11171 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11172 case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op1);
11173 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11174 case MVT::v16f16: return fastEmit_ISD_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
11175 case MVT::v32f16: return fastEmit_ISD_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
11176 case MVT::v8bf16: return fastEmit_ISD_FADD_MVT_v8bf16_rr(RetVT, Op0, Op1);
11177 case MVT::v16bf16: return fastEmit_ISD_FADD_MVT_v16bf16_rr(RetVT, Op0, Op1);
11178 case MVT::v32bf16: return fastEmit_ISD_FADD_MVT_v32bf16_rr(RetVT, Op0, Op1);
11179 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11180 case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
11181 case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
11182 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11183 case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
11184 case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
11185 default: return Register();
11186 }
11187}
11188
11189// FastEmit functions for ISD::FDIV.
11190
11191Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11192 if (RetVT.SimpleTy != MVT::f16)
11193 return Register();
11194 if ((Subtarget->hasFP16())) {
11195 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11196 }
11197 return Register();
11198}
11199
11200Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11201 if (RetVT.SimpleTy != MVT::f32)
11202 return Register();
11203 if ((Subtarget->hasAVX512())) {
11204 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11205 }
11206 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11207 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11208 }
11209 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11210 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11211 }
11212 if ((!Subtarget->hasSSE1())) {
11213 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11214 }
11215 return Register();
11216}
11217
11218Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11219 if (RetVT.SimpleTy != MVT::f64)
11220 return Register();
11221 if ((Subtarget->hasAVX512())) {
11222 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11223 }
11224 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11225 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11226 }
11227 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11228 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11229 }
11230 if ((!Subtarget->hasSSE2())) {
11231 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11232 }
11233 return Register();
11234}
11235
11236Register fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11237 if (RetVT.SimpleTy != MVT::f80)
11238 return Register();
11239 if ((Subtarget->hasX87())) {
11240 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11241 }
11242 return Register();
11243}
11244
11245Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11246 if (RetVT.SimpleTy != MVT::v8f16)
11247 return Register();
11248 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11249 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11250 }
11251 return Register();
11252}
11253
11254Register fastEmit_ISD_FDIV_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11255 if (RetVT.SimpleTy != MVT::v16f16)
11256 return Register();
11257 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11258 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11259 }
11260 return Register();
11261}
11262
11263Register fastEmit_ISD_FDIV_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11264 if (RetVT.SimpleTy != MVT::v32f16)
11265 return Register();
11266 if ((Subtarget->hasFP16())) {
11267 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11268 }
11269 return Register();
11270}
11271
11272Register fastEmit_ISD_FDIV_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11273 if (RetVT.SimpleTy != MVT::v8bf16)
11274 return Register();
11275 if ((Subtarget->hasAVX10_2())) {
11276 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11277 }
11278 return Register();
11279}
11280
11281Register fastEmit_ISD_FDIV_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11282 if (RetVT.SimpleTy != MVT::v16bf16)
11283 return Register();
11284 if ((Subtarget->hasAVX10_2())) {
11285 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11286 }
11287 return Register();
11288}
11289
11290Register fastEmit_ISD_FDIV_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11291 if (RetVT.SimpleTy != MVT::v32bf16)
11292 return Register();
11293 if ((Subtarget->hasAVX10_2())) {
11294 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11295 }
11296 return Register();
11297}
11298
11299Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11300 if (RetVT.SimpleTy != MVT::v4f32)
11301 return Register();
11302 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11303 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11304 }
11305 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11306 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11307 }
11308 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11309 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11310 }
11311 return Register();
11312}
11313
11314Register fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11315 if (RetVT.SimpleTy != MVT::v8f32)
11316 return Register();
11317 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11318 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11319 }
11320 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11321 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11322 }
11323 return Register();
11324}
11325
11326Register fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11327 if (RetVT.SimpleTy != MVT::v16f32)
11328 return Register();
11329 if ((Subtarget->hasAVX512())) {
11330 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11331 }
11332 return Register();
11333}
11334
11335Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11336 if (RetVT.SimpleTy != MVT::v2f64)
11337 return Register();
11338 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11339 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11340 }
11341 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11342 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11343 }
11344 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11345 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11346 }
11347 return Register();
11348}
11349
11350Register fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11351 if (RetVT.SimpleTy != MVT::v4f64)
11352 return Register();
11353 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11354 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11355 }
11356 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11357 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11358 }
11359 return Register();
11360}
11361
11362Register fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11363 if (RetVT.SimpleTy != MVT::v8f64)
11364 return Register();
11365 if ((Subtarget->hasAVX512())) {
11366 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11367 }
11368 return Register();
11369}
11370
11371Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11372 switch (VT.SimpleTy) {
11373 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11374 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11375 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11376 case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
11377 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11378 case MVT::v16f16: return fastEmit_ISD_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
11379 case MVT::v32f16: return fastEmit_ISD_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
11380 case MVT::v8bf16: return fastEmit_ISD_FDIV_MVT_v8bf16_rr(RetVT, Op0, Op1);
11381 case MVT::v16bf16: return fastEmit_ISD_FDIV_MVT_v16bf16_rr(RetVT, Op0, Op1);
11382 case MVT::v32bf16: return fastEmit_ISD_FDIV_MVT_v32bf16_rr(RetVT, Op0, Op1);
11383 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11384 case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
11385 case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
11386 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11387 case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
11388 case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
11389 default: return Register();
11390 }
11391}
11392
11393// FastEmit functions for ISD::FMUL.
11394
11395Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11396 if (RetVT.SimpleTy != MVT::f16)
11397 return Register();
11398 if ((Subtarget->hasFP16())) {
11399 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11400 }
11401 return Register();
11402}
11403
11404Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11405 if (RetVT.SimpleTy != MVT::f32)
11406 return Register();
11407 if ((Subtarget->hasAVX512())) {
11408 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11409 }
11410 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11411 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11412 }
11413 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11414 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11415 }
11416 if ((!Subtarget->hasSSE1())) {
11417 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11418 }
11419 return Register();
11420}
11421
11422Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11423 if (RetVT.SimpleTy != MVT::f64)
11424 return Register();
11425 if ((Subtarget->hasAVX512())) {
11426 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11427 }
11428 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11429 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11430 }
11431 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11432 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11433 }
11434 if ((!Subtarget->hasSSE2())) {
11435 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11436 }
11437 return Register();
11438}
11439
11440Register fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11441 if (RetVT.SimpleTy != MVT::f80)
11442 return Register();
11443 if ((Subtarget->hasX87())) {
11444 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11445 }
11446 return Register();
11447}
11448
11449Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11450 if (RetVT.SimpleTy != MVT::v8f16)
11451 return Register();
11452 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11453 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11454 }
11455 return Register();
11456}
11457
11458Register fastEmit_ISD_FMUL_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11459 if (RetVT.SimpleTy != MVT::v16f16)
11460 return Register();
11461 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11462 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11463 }
11464 return Register();
11465}
11466
11467Register fastEmit_ISD_FMUL_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11468 if (RetVT.SimpleTy != MVT::v32f16)
11469 return Register();
11470 if ((Subtarget->hasFP16())) {
11471 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11472 }
11473 return Register();
11474}
11475
11476Register fastEmit_ISD_FMUL_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11477 if (RetVT.SimpleTy != MVT::v8bf16)
11478 return Register();
11479 if ((Subtarget->hasAVX10_2())) {
11480 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11481 }
11482 return Register();
11483}
11484
11485Register fastEmit_ISD_FMUL_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11486 if (RetVT.SimpleTy != MVT::v16bf16)
11487 return Register();
11488 if ((Subtarget->hasAVX10_2())) {
11489 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11490 }
11491 return Register();
11492}
11493
11494Register fastEmit_ISD_FMUL_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11495 if (RetVT.SimpleTy != MVT::v32bf16)
11496 return Register();
11497 if ((Subtarget->hasAVX10_2())) {
11498 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11499 }
11500 return Register();
11501}
11502
11503Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11504 if (RetVT.SimpleTy != MVT::v4f32)
11505 return Register();
11506 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11507 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11508 }
11509 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11510 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11511 }
11512 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11513 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11514 }
11515 return Register();
11516}
11517
11518Register fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11519 if (RetVT.SimpleTy != MVT::v8f32)
11520 return Register();
11521 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11522 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11523 }
11524 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11525 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11526 }
11527 return Register();
11528}
11529
11530Register fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11531 if (RetVT.SimpleTy != MVT::v16f32)
11532 return Register();
11533 if ((Subtarget->hasAVX512())) {
11534 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11535 }
11536 return Register();
11537}
11538
11539Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11540 if (RetVT.SimpleTy != MVT::v2f64)
11541 return Register();
11542 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11543 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11544 }
11545 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11546 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11547 }
11548 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11549 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11550 }
11551 return Register();
11552}
11553
11554Register fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11555 if (RetVT.SimpleTy != MVT::v4f64)
11556 return Register();
11557 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11558 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11559 }
11560 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11561 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11562 }
11563 return Register();
11564}
11565
11566Register fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11567 if (RetVT.SimpleTy != MVT::v8f64)
11568 return Register();
11569 if ((Subtarget->hasAVX512())) {
11570 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11571 }
11572 return Register();
11573}
11574
11575Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11576 switch (VT.SimpleTy) {
11577 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11578 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11579 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11580 case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
11581 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11582 case MVT::v16f16: return fastEmit_ISD_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
11583 case MVT::v32f16: return fastEmit_ISD_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
11584 case MVT::v8bf16: return fastEmit_ISD_FMUL_MVT_v8bf16_rr(RetVT, Op0, Op1);
11585 case MVT::v16bf16: return fastEmit_ISD_FMUL_MVT_v16bf16_rr(RetVT, Op0, Op1);
11586 case MVT::v32bf16: return fastEmit_ISD_FMUL_MVT_v32bf16_rr(RetVT, Op0, Op1);
11587 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11588 case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
11589 case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
11590 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11591 case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
11592 case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
11593 default: return Register();
11594 }
11595}
11596
11597// FastEmit functions for ISD::FSUB.
11598
11599Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11600 if (RetVT.SimpleTy != MVT::f16)
11601 return Register();
11602 if ((Subtarget->hasFP16())) {
11603 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11604 }
11605 return Register();
11606}
11607
11608Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11609 if (RetVT.SimpleTy != MVT::f32)
11610 return Register();
11611 if ((Subtarget->hasAVX512())) {
11612 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11613 }
11614 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11615 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11616 }
11617 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11618 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11619 }
11620 if ((!Subtarget->hasSSE1())) {
11621 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11622 }
11623 return Register();
11624}
11625
11626Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11627 if (RetVT.SimpleTy != MVT::f64)
11628 return Register();
11629 if ((Subtarget->hasAVX512())) {
11630 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11631 }
11632 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11633 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11634 }
11635 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11636 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11637 }
11638 if ((!Subtarget->hasSSE2())) {
11639 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11640 }
11641 return Register();
11642}
11643
11644Register fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11645 if (RetVT.SimpleTy != MVT::f80)
11646 return Register();
11647 if ((Subtarget->hasX87())) {
11648 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11649 }
11650 return Register();
11651}
11652
11653Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11654 if (RetVT.SimpleTy != MVT::v8f16)
11655 return Register();
11656 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11657 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11658 }
11659 return Register();
11660}
11661
11662Register fastEmit_ISD_FSUB_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11663 if (RetVT.SimpleTy != MVT::v16f16)
11664 return Register();
11665 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11666 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11667 }
11668 return Register();
11669}
11670
11671Register fastEmit_ISD_FSUB_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11672 if (RetVT.SimpleTy != MVT::v32f16)
11673 return Register();
11674 if ((Subtarget->hasFP16())) {
11675 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11676 }
11677 return Register();
11678}
11679
11680Register fastEmit_ISD_FSUB_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11681 if (RetVT.SimpleTy != MVT::v8bf16)
11682 return Register();
11683 if ((Subtarget->hasAVX10_2())) {
11684 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11685 }
11686 return Register();
11687}
11688
11689Register fastEmit_ISD_FSUB_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11690 if (RetVT.SimpleTy != MVT::v16bf16)
11691 return Register();
11692 if ((Subtarget->hasAVX10_2())) {
11693 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11694 }
11695 return Register();
11696}
11697
11698Register fastEmit_ISD_FSUB_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11699 if (RetVT.SimpleTy != MVT::v32bf16)
11700 return Register();
11701 if ((Subtarget->hasAVX10_2())) {
11702 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11703 }
11704 return Register();
11705}
11706
11707Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11708 if (RetVT.SimpleTy != MVT::v4f32)
11709 return Register();
11710 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11711 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11712 }
11713 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11714 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11715 }
11716 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11717 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11718 }
11719 return Register();
11720}
11721
11722Register fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11723 if (RetVT.SimpleTy != MVT::v8f32)
11724 return Register();
11725 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11726 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11727 }
11728 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11729 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11730 }
11731 return Register();
11732}
11733
11734Register fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11735 if (RetVT.SimpleTy != MVT::v16f32)
11736 return Register();
11737 if ((Subtarget->hasAVX512())) {
11738 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11739 }
11740 return Register();
11741}
11742
11743Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11744 if (RetVT.SimpleTy != MVT::v2f64)
11745 return Register();
11746 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11747 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11748 }
11749 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11750 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11751 }
11752 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11753 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11754 }
11755 return Register();
11756}
11757
11758Register fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11759 if (RetVT.SimpleTy != MVT::v4f64)
11760 return Register();
11761 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11762 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11763 }
11764 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11765 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11766 }
11767 return Register();
11768}
11769
11770Register fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11771 if (RetVT.SimpleTy != MVT::v8f64)
11772 return Register();
11773 if ((Subtarget->hasAVX512())) {
11774 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11775 }
11776 return Register();
11777}
11778
11779Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11780 switch (VT.SimpleTy) {
11781 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11782 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11783 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11784 case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
11785 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11786 case MVT::v16f16: return fastEmit_ISD_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
11787 case MVT::v32f16: return fastEmit_ISD_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
11788 case MVT::v8bf16: return fastEmit_ISD_FSUB_MVT_v8bf16_rr(RetVT, Op0, Op1);
11789 case MVT::v16bf16: return fastEmit_ISD_FSUB_MVT_v16bf16_rr(RetVT, Op0, Op1);
11790 case MVT::v32bf16: return fastEmit_ISD_FSUB_MVT_v32bf16_rr(RetVT, Op0, Op1);
11791 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11792 case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
11793 case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
11794 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11795 case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
11796 case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
11797 default: return Register();
11798 }
11799}
11800
11801// FastEmit functions for ISD::MUL.
11802
11803Register fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
11804 if (RetVT.SimpleTy != MVT::i8)
11805 return Register();
11806 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::AL).addReg(RegNo: Op0);
11807 return fastEmitInst_r(MachineInstOpcode: X86::MUL8r, RC: &X86::GR8RegClass, Op0: Op1);
11808}
11809
11810Register fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
11811 if (RetVT.SimpleTy != MVT::i16)
11812 return Register();
11813 if ((Subtarget->hasNDD())) {
11814 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
11815 }
11816 if ((!Subtarget->hasNDD())) {
11817 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr, RC: &X86::GR16RegClass, Op0, Op1);
11818 }
11819 return Register();
11820}
11821
11822Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11823 if (RetVT.SimpleTy != MVT::i32)
11824 return Register();
11825 if ((Subtarget->hasNDD())) {
11826 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
11827 }
11828 if ((!Subtarget->hasNDD())) {
11829 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr, RC: &X86::GR32RegClass, Op0, Op1);
11830 }
11831 return Register();
11832}
11833
11834Register fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11835 if (RetVT.SimpleTy != MVT::i64)
11836 return Register();
11837 if ((Subtarget->hasNDD())) {
11838 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
11839 }
11840 if ((!Subtarget->hasNDD())) {
11841 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr, RC: &X86::GR64RegClass, Op0, Op1);
11842 }
11843 return Register();
11844}
11845
11846Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11847 if (RetVT.SimpleTy != MVT::v8i16)
11848 return Register();
11849 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11850 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11851 }
11852 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11853 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
11854 }
11855 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11856 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
11857 }
11858 return Register();
11859}
11860
11861Register fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
11862 if (RetVT.SimpleTy != MVT::v16i16)
11863 return Register();
11864 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11865 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11866 }
11867 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11868 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWYrr, RC: &X86::VR256RegClass, Op0, Op1);
11869 }
11870 return Register();
11871}
11872
11873Register fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
11874 if (RetVT.SimpleTy != MVT::v32i16)
11875 return Register();
11876 if ((Subtarget->hasBWI())) {
11877 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11878 }
11879 return Register();
11880}
11881
11882Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11883 if (RetVT.SimpleTy != MVT::v4i32)
11884 return Register();
11885 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11886 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11887 }
11888 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
11889 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
11890 }
11891 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11892 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
11893 }
11894 return Register();
11895}
11896
11897Register fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
11898 if (RetVT.SimpleTy != MVT::v8i32)
11899 return Register();
11900 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11901 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11902 }
11903 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11904 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11905 }
11906 return Register();
11907}
11908
11909Register fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
11910 if (RetVT.SimpleTy != MVT::v16i32)
11911 return Register();
11912 if ((Subtarget->hasAVX512())) {
11913 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11914 }
11915 return Register();
11916}
11917
11918Register fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11919 if (RetVT.SimpleTy != MVT::v2i64)
11920 return Register();
11921 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
11922 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11923 }
11924 return Register();
11925}
11926
11927Register fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
11928 if (RetVT.SimpleTy != MVT::v4i64)
11929 return Register();
11930 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
11931 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11932 }
11933 return Register();
11934}
11935
11936Register fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
11937 if (RetVT.SimpleTy != MVT::v8i64)
11938 return Register();
11939 if ((Subtarget->hasDQI())) {
11940 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZrr, RC: &X86::VR512RegClass, Op0, Op1);
11941 }
11942 return Register();
11943}
11944
11945Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11946 switch (VT.SimpleTy) {
11947 case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op1);
11948 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op1);
11949 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
11950 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op1);
11951 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
11952 case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op1);
11953 case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op1);
11954 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
11955 case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op1);
11956 case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op1);
11957 case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op1);
11958 case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op1);
11959 case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op1);
11960 default: return Register();
11961 }
11962}
11963
11964// FastEmit functions for ISD::MULHS.
11965
11966Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11967 if (RetVT.SimpleTy != MVT::v8i16)
11968 return Register();
11969 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11970 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11971 }
11972 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11973 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
11974 }
11975 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11976 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
11977 }
11978 return Register();
11979}
11980
11981Register fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
11982 if (RetVT.SimpleTy != MVT::v16i16)
11983 return Register();
11984 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11985 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11986 }
11987 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11988 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWYrr, RC: &X86::VR256RegClass, Op0, Op1);
11989 }
11990 return Register();
11991}
11992
11993Register fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
11994 if (RetVT.SimpleTy != MVT::v32i16)
11995 return Register();
11996 if ((Subtarget->hasBWI())) {
11997 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11998 }
11999 return Register();
12000}
12001
12002Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12003 switch (VT.SimpleTy) {
12004 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
12005 case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op1);
12006 case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op1);
12007 default: return Register();
12008 }
12009}
12010
12011// FastEmit functions for ISD::MULHU.
12012
12013Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12014 if (RetVT.SimpleTy != MVT::v8i16)
12015 return Register();
12016 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12017 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12018 }
12019 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12020 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12021 }
12022 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12023 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12024 }
12025 return Register();
12026}
12027
12028Register fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12029 if (RetVT.SimpleTy != MVT::v16i16)
12030 return Register();
12031 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12032 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12033 }
12034 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12035 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12036 }
12037 return Register();
12038}
12039
12040Register fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12041 if (RetVT.SimpleTy != MVT::v32i16)
12042 return Register();
12043 if ((Subtarget->hasBWI())) {
12044 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12045 }
12046 return Register();
12047}
12048
12049Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12050 switch (VT.SimpleTy) {
12051 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
12052 case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op1);
12053 case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op1);
12054 default: return Register();
12055 }
12056}
12057
12058// FastEmit functions for ISD::OR.
12059
12060Register fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12061 if (RetVT.SimpleTy != MVT::i8)
12062 return Register();
12063 if ((Subtarget->hasNDD())) {
12064 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
12065 }
12066 if ((!Subtarget->hasNDD())) {
12067 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr, RC: &X86::GR8RegClass, Op0, Op1);
12068 }
12069 return Register();
12070}
12071
12072Register fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
12073 if (RetVT.SimpleTy != MVT::i16)
12074 return Register();
12075 if ((Subtarget->hasNDD())) {
12076 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
12077 }
12078 if ((!Subtarget->hasNDD())) {
12079 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr, RC: &X86::GR16RegClass, Op0, Op1);
12080 }
12081 return Register();
12082}
12083
12084Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12085 if (RetVT.SimpleTy != MVT::i32)
12086 return Register();
12087 if ((Subtarget->hasNDD())) {
12088 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
12089 }
12090 if ((!Subtarget->hasNDD())) {
12091 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr, RC: &X86::GR32RegClass, Op0, Op1);
12092 }
12093 return Register();
12094}
12095
12096Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12097 if (RetVT.SimpleTy != MVT::i64)
12098 return Register();
12099 if ((Subtarget->hasNDD())) {
12100 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
12101 }
12102 if ((!Subtarget->hasNDD())) {
12103 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr, RC: &X86::GR64RegClass, Op0, Op1);
12104 }
12105 return Register();
12106}
12107
12108Register fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
12109 if (RetVT.SimpleTy != MVT::v8i1)
12110 return Register();
12111 if ((Subtarget->hasDQI())) {
12112 return fastEmitInst_rr(MachineInstOpcode: X86::KORBkk, RC: &X86::VK8RegClass, Op0, Op1);
12113 }
12114 return Register();
12115}
12116
12117Register fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
12118 if (RetVT.SimpleTy != MVT::v16i1)
12119 return Register();
12120 if ((Subtarget->hasAVX512())) {
12121 return fastEmitInst_rr(MachineInstOpcode: X86::KORWkk, RC: &X86::VK16RegClass, Op0, Op1);
12122 }
12123 return Register();
12124}
12125
12126Register fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
12127 if (RetVT.SimpleTy != MVT::v32i1)
12128 return Register();
12129 if ((Subtarget->hasBWI())) {
12130 return fastEmitInst_rr(MachineInstOpcode: X86::KORDkk, RC: &X86::VK32RegClass, Op0, Op1);
12131 }
12132 return Register();
12133}
12134
12135Register fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
12136 if (RetVT.SimpleTy != MVT::v64i1)
12137 return Register();
12138 if ((Subtarget->hasBWI())) {
12139 return fastEmitInst_rr(MachineInstOpcode: X86::KORQkk, RC: &X86::VK64RegClass, Op0, Op1);
12140 }
12141 return Register();
12142}
12143
12144Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12145 if (RetVT.SimpleTy != MVT::v16i8)
12146 return Register();
12147 if ((Subtarget->hasVLX())) {
12148 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12149 }
12150 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12151 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12152 }
12153 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12154 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12155 }
12156 return Register();
12157}
12158
12159Register fastEmit_ISD_OR_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12160 if (RetVT.SimpleTy != MVT::v32i8)
12161 return Register();
12162 if ((Subtarget->hasVLX())) {
12163 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12164 }
12165 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12166 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12167 }
12168 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12169 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12170 }
12171 return Register();
12172}
12173
12174Register fastEmit_ISD_OR_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12175 if (RetVT.SimpleTy != MVT::v64i8)
12176 return Register();
12177 if ((Subtarget->hasAVX512())) {
12178 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12179 }
12180 return Register();
12181}
12182
12183Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12184 if (RetVT.SimpleTy != MVT::v8i16)
12185 return Register();
12186 if ((Subtarget->hasVLX())) {
12187 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12188 }
12189 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12190 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12191 }
12192 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12193 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12194 }
12195 return Register();
12196}
12197
12198Register fastEmit_ISD_OR_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12199 if (RetVT.SimpleTy != MVT::v16i16)
12200 return Register();
12201 if ((Subtarget->hasVLX())) {
12202 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12203 }
12204 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12205 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12206 }
12207 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12208 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12209 }
12210 return Register();
12211}
12212
12213Register fastEmit_ISD_OR_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12214 if (RetVT.SimpleTy != MVT::v32i16)
12215 return Register();
12216 if ((Subtarget->hasAVX512())) {
12217 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12218 }
12219 return Register();
12220}
12221
12222Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12223 if (RetVT.SimpleTy != MVT::v4i32)
12224 return Register();
12225 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12226 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12227 }
12228 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12229 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12230 }
12231 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12232 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12233 }
12234 return Register();
12235}
12236
12237Register fastEmit_ISD_OR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12238 if (RetVT.SimpleTy != MVT::v8i32)
12239 return Register();
12240 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12241 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12242 }
12243 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12244 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12245 }
12246 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12247 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12248 }
12249 return Register();
12250}
12251
12252Register fastEmit_ISD_OR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12253 if (RetVT.SimpleTy != MVT::v16i32)
12254 return Register();
12255 if ((Subtarget->hasAVX512())) {
12256 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12257 }
12258 return Register();
12259}
12260
12261Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12262 if (RetVT.SimpleTy != MVT::v2i64)
12263 return Register();
12264 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12265 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12266 }
12267 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12268 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12269 }
12270 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12271 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12272 }
12273 return Register();
12274}
12275
12276Register fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12277 if (RetVT.SimpleTy != MVT::v4i64)
12278 return Register();
12279 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12280 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12281 }
12282 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12283 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12284 }
12285 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12286 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12287 }
12288 return Register();
12289}
12290
12291Register fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12292 if (RetVT.SimpleTy != MVT::v8i64)
12293 return Register();
12294 if ((Subtarget->hasAVX512())) {
12295 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12296 }
12297 return Register();
12298}
12299
12300Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12301 switch (VT.SimpleTy) {
12302 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op1);
12303 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op1);
12304 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12305 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12306 case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op1);
12307 case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op1);
12308 case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op1);
12309 case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op1);
12310 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12311 case MVT::v32i8: return fastEmit_ISD_OR_MVT_v32i8_rr(RetVT, Op0, Op1);
12312 case MVT::v64i8: return fastEmit_ISD_OR_MVT_v64i8_rr(RetVT, Op0, Op1);
12313 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12314 case MVT::v16i16: return fastEmit_ISD_OR_MVT_v16i16_rr(RetVT, Op0, Op1);
12315 case MVT::v32i16: return fastEmit_ISD_OR_MVT_v32i16_rr(RetVT, Op0, Op1);
12316 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12317 case MVT::v8i32: return fastEmit_ISD_OR_MVT_v8i32_rr(RetVT, Op0, Op1);
12318 case MVT::v16i32: return fastEmit_ISD_OR_MVT_v16i32_rr(RetVT, Op0, Op1);
12319 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12320 case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op1);
12321 case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op1);
12322 default: return Register();
12323 }
12324}
12325
12326// FastEmit functions for ISD::ROTL.
12327
12328Register fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12329 if (RetVT.SimpleTy != MVT::i8)
12330 return Register();
12331 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12332 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12333 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL_ND, RC: &X86::GR8RegClass, Op0);
12334 }
12335 if ((!Subtarget->hasNDD())) {
12336 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12337 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL, RC: &X86::GR8RegClass, Op0);
12338 }
12339 return Register();
12340}
12341
12342Register fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12343 if (RetVT.SimpleTy != MVT::v16i8)
12344 return Register();
12345 if ((Subtarget->hasXOP())) {
12346 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTBrr, RC: &X86::VR128RegClass, Op0, Op1);
12347 }
12348 return Register();
12349}
12350
12351Register fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12352 if (RetVT.SimpleTy != MVT::v8i16)
12353 return Register();
12354 if ((Subtarget->hasXOP())) {
12355 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTWrr, RC: &X86::VR128RegClass, Op0, Op1);
12356 }
12357 return Register();
12358}
12359
12360Register fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12361 if (RetVT.SimpleTy != MVT::v4i32)
12362 return Register();
12363 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12364 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12365 }
12366 if ((Subtarget->hasXOP())) {
12367 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTDrr, RC: &X86::VR128RegClass, Op0, Op1);
12368 }
12369 return Register();
12370}
12371
12372Register fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12373 if (RetVT.SimpleTy != MVT::v8i32)
12374 return Register();
12375 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12376 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12377 }
12378 return Register();
12379}
12380
12381Register fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12382 if (RetVT.SimpleTy != MVT::v16i32)
12383 return Register();
12384 if ((Subtarget->hasAVX512())) {
12385 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12386 }
12387 return Register();
12388}
12389
12390Register fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12391 if (RetVT.SimpleTy != MVT::v2i64)
12392 return Register();
12393 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12394 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12395 }
12396 if ((Subtarget->hasXOP())) {
12397 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTQrr, RC: &X86::VR128RegClass, Op0, Op1);
12398 }
12399 return Register();
12400}
12401
12402Register fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12403 if (RetVT.SimpleTy != MVT::v4i64)
12404 return Register();
12405 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12406 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12407 }
12408 return Register();
12409}
12410
12411Register fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12412 if (RetVT.SimpleTy != MVT::v8i64)
12413 return Register();
12414 if ((Subtarget->hasAVX512())) {
12415 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12416 }
12417 return Register();
12418}
12419
12420Register fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12421 switch (VT.SimpleTy) {
12422 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op1);
12423 case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op1);
12424 case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op1);
12425 case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op1);
12426 case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op1);
12427 case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op1);
12428 case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op1);
12429 case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op1);
12430 case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op1);
12431 default: return Register();
12432 }
12433}
12434
12435// FastEmit functions for ISD::ROTR.
12436
12437Register fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12438 if (RetVT.SimpleTy != MVT::i8)
12439 return Register();
12440 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12441 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12442 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL_ND, RC: &X86::GR8RegClass, Op0);
12443 }
12444 if ((!Subtarget->hasNDD())) {
12445 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12446 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL, RC: &X86::GR8RegClass, Op0);
12447 }
12448 return Register();
12449}
12450
12451Register fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12452 if (RetVT.SimpleTy != MVT::v4i32)
12453 return Register();
12454 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12455 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12456 }
12457 return Register();
12458}
12459
12460Register fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12461 if (RetVT.SimpleTy != MVT::v8i32)
12462 return Register();
12463 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12464 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12465 }
12466 return Register();
12467}
12468
12469Register fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12470 if (RetVT.SimpleTy != MVT::v16i32)
12471 return Register();
12472 if ((Subtarget->hasAVX512())) {
12473 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12474 }
12475 return Register();
12476}
12477
12478Register fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12479 if (RetVT.SimpleTy != MVT::v2i64)
12480 return Register();
12481 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12482 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12483 }
12484 return Register();
12485}
12486
12487Register fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12488 if (RetVT.SimpleTy != MVT::v4i64)
12489 return Register();
12490 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12491 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12492 }
12493 return Register();
12494}
12495
12496Register fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12497 if (RetVT.SimpleTy != MVT::v8i64)
12498 return Register();
12499 if ((Subtarget->hasAVX512())) {
12500 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12501 }
12502 return Register();
12503}
12504
12505Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12506 switch (VT.SimpleTy) {
12507 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op1);
12508 case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op1);
12509 case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op1);
12510 case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op1);
12511 case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op1);
12512 case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op1);
12513 case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op1);
12514 default: return Register();
12515 }
12516}
12517
12518// FastEmit functions for ISD::SADDSAT.
12519
12520Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12521 if (RetVT.SimpleTy != MVT::v16i8)
12522 return Register();
12523 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12524 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12525 }
12526 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12527 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12528 }
12529 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12530 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12531 }
12532 return Register();
12533}
12534
12535Register fastEmit_ISD_SADDSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12536 if (RetVT.SimpleTy != MVT::v32i8)
12537 return Register();
12538 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12539 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12540 }
12541 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12542 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12543 }
12544 return Register();
12545}
12546
12547Register fastEmit_ISD_SADDSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12548 if (RetVT.SimpleTy != MVT::v64i8)
12549 return Register();
12550 if ((Subtarget->hasBWI())) {
12551 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12552 }
12553 return Register();
12554}
12555
12556Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12557 if (RetVT.SimpleTy != MVT::v8i16)
12558 return Register();
12559 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12560 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12561 }
12562 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12563 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12564 }
12565 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12566 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12567 }
12568 return Register();
12569}
12570
12571Register fastEmit_ISD_SADDSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12572 if (RetVT.SimpleTy != MVT::v16i16)
12573 return Register();
12574 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12575 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12576 }
12577 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12578 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12579 }
12580 return Register();
12581}
12582
12583Register fastEmit_ISD_SADDSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12584 if (RetVT.SimpleTy != MVT::v32i16)
12585 return Register();
12586 if ((Subtarget->hasBWI())) {
12587 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12588 }
12589 return Register();
12590}
12591
12592Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12593 switch (VT.SimpleTy) {
12594 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12595 case MVT::v32i8: return fastEmit_ISD_SADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
12596 case MVT::v64i8: return fastEmit_ISD_SADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
12597 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12598 case MVT::v16i16: return fastEmit_ISD_SADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
12599 case MVT::v32i16: return fastEmit_ISD_SADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
12600 default: return Register();
12601 }
12602}
12603
12604// FastEmit functions for ISD::SHL.
12605
12606Register fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12607 if (RetVT.SimpleTy != MVT::i8)
12608 return Register();
12609 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12610 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12611 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL_ND, RC: &X86::GR8RegClass, Op0);
12612 }
12613 if ((!Subtarget->hasNDD())) {
12614 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12615 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL, RC: &X86::GR8RegClass, Op0);
12616 }
12617 return Register();
12618}
12619
12620Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12621 switch (VT.SimpleTy) {
12622 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op1);
12623 default: return Register();
12624 }
12625}
12626
12627// FastEmit functions for ISD::SMAX.
12628
12629Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12630 if (RetVT.SimpleTy != MVT::v16i8)
12631 return Register();
12632 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12633 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12634 }
12635 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12636 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12637 }
12638 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12639 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12640 }
12641 return Register();
12642}
12643
12644Register fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12645 if (RetVT.SimpleTy != MVT::v32i8)
12646 return Register();
12647 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12648 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12649 }
12650 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12651 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12652 }
12653 return Register();
12654}
12655
12656Register fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12657 if (RetVT.SimpleTy != MVT::v64i8)
12658 return Register();
12659 if ((Subtarget->hasBWI())) {
12660 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12661 }
12662 return Register();
12663}
12664
12665Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12666 if (RetVT.SimpleTy != MVT::v8i16)
12667 return Register();
12668 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12669 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12670 }
12671 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12672 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12673 }
12674 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12675 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12676 }
12677 return Register();
12678}
12679
12680Register fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12681 if (RetVT.SimpleTy != MVT::v16i16)
12682 return Register();
12683 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12684 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12685 }
12686 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12687 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12688 }
12689 return Register();
12690}
12691
12692Register fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12693 if (RetVT.SimpleTy != MVT::v32i16)
12694 return Register();
12695 if ((Subtarget->hasBWI())) {
12696 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12697 }
12698 return Register();
12699}
12700
12701Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12702 if (RetVT.SimpleTy != MVT::v4i32)
12703 return Register();
12704 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12705 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12706 }
12707 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12708 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12709 }
12710 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12711 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12712 }
12713 return Register();
12714}
12715
12716Register fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12717 if (RetVT.SimpleTy != MVT::v8i32)
12718 return Register();
12719 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12720 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12721 }
12722 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12723 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12724 }
12725 return Register();
12726}
12727
12728Register fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12729 if (RetVT.SimpleTy != MVT::v16i32)
12730 return Register();
12731 if ((Subtarget->hasAVX512())) {
12732 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12733 }
12734 return Register();
12735}
12736
12737Register fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12738 if (RetVT.SimpleTy != MVT::v2i64)
12739 return Register();
12740 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12741 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12742 }
12743 return Register();
12744}
12745
12746Register fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12747 if (RetVT.SimpleTy != MVT::v4i64)
12748 return Register();
12749 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12750 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12751 }
12752 return Register();
12753}
12754
12755Register fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12756 if (RetVT.SimpleTy != MVT::v8i64)
12757 return Register();
12758 if ((Subtarget->hasAVX512())) {
12759 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12760 }
12761 return Register();
12762}
12763
12764Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12765 switch (VT.SimpleTy) {
12766 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12767 case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
12768 case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
12769 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12770 case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
12771 case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
12772 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12773 case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
12774 case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
12775 case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
12776 case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
12777 case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
12778 default: return Register();
12779 }
12780}
12781
12782// FastEmit functions for ISD::SMIN.
12783
12784Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12785 if (RetVT.SimpleTy != MVT::v16i8)
12786 return Register();
12787 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12788 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12789 }
12790 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12791 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12792 }
12793 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12794 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12795 }
12796 return Register();
12797}
12798
12799Register fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12800 if (RetVT.SimpleTy != MVT::v32i8)
12801 return Register();
12802 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12803 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12804 }
12805 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12806 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
12807 }
12808 return Register();
12809}
12810
12811Register fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12812 if (RetVT.SimpleTy != MVT::v64i8)
12813 return Register();
12814 if ((Subtarget->hasBWI())) {
12815 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
12816 }
12817 return Register();
12818}
12819
12820Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12821 if (RetVT.SimpleTy != MVT::v8i16)
12822 return Register();
12823 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12824 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12825 }
12826 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12827 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12828 }
12829 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12830 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
12831 }
12832 return Register();
12833}
12834
12835Register fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12836 if (RetVT.SimpleTy != MVT::v16i16)
12837 return Register();
12838 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12839 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12840 }
12841 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12842 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12843 }
12844 return Register();
12845}
12846
12847Register fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12848 if (RetVT.SimpleTy != MVT::v32i16)
12849 return Register();
12850 if ((Subtarget->hasBWI())) {
12851 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12852 }
12853 return Register();
12854}
12855
12856Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12857 if (RetVT.SimpleTy != MVT::v4i32)
12858 return Register();
12859 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12860 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12861 }
12862 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12863 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12864 }
12865 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12866 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
12867 }
12868 return Register();
12869}
12870
12871Register fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12872 if (RetVT.SimpleTy != MVT::v8i32)
12873 return Register();
12874 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12875 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12876 }
12877 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12878 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12879 }
12880 return Register();
12881}
12882
12883Register fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12884 if (RetVT.SimpleTy != MVT::v16i32)
12885 return Register();
12886 if ((Subtarget->hasAVX512())) {
12887 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12888 }
12889 return Register();
12890}
12891
12892Register fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12893 if (RetVT.SimpleTy != MVT::v2i64)
12894 return Register();
12895 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12896 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12897 }
12898 return Register();
12899}
12900
12901Register fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12902 if (RetVT.SimpleTy != MVT::v4i64)
12903 return Register();
12904 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12905 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12906 }
12907 return Register();
12908}
12909
12910Register fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12911 if (RetVT.SimpleTy != MVT::v8i64)
12912 return Register();
12913 if ((Subtarget->hasAVX512())) {
12914 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12915 }
12916 return Register();
12917}
12918
12919Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12920 switch (VT.SimpleTy) {
12921 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12922 case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
12923 case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
12924 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12925 case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
12926 case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
12927 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12928 case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
12929 case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
12930 case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
12931 case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
12932 case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
12933 default: return Register();
12934 }
12935}
12936
12937// FastEmit functions for ISD::SRA.
12938
12939Register fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12940 if (RetVT.SimpleTy != MVT::i8)
12941 return Register();
12942 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12943 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12944 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL_ND, RC: &X86::GR8RegClass, Op0);
12945 }
12946 if ((!Subtarget->hasNDD())) {
12947 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12948 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL, RC: &X86::GR8RegClass, Op0);
12949 }
12950 return Register();
12951}
12952
12953Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12954 switch (VT.SimpleTy) {
12955 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op1);
12956 default: return Register();
12957 }
12958}
12959
12960// FastEmit functions for ISD::SRL.
12961
12962Register fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12963 if (RetVT.SimpleTy != MVT::i8)
12964 return Register();
12965 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12966 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12967 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL_ND, RC: &X86::GR8RegClass, Op0);
12968 }
12969 if ((!Subtarget->hasNDD())) {
12970 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12971 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL, RC: &X86::GR8RegClass, Op0);
12972 }
12973 return Register();
12974}
12975
12976Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12977 switch (VT.SimpleTy) {
12978 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op1);
12979 default: return Register();
12980 }
12981}
12982
12983// FastEmit functions for ISD::SSUBSAT.
12984
12985Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12986 if (RetVT.SimpleTy != MVT::v16i8)
12987 return Register();
12988 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12989 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12990 }
12991 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12992 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12993 }
12994 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12995 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
12996 }
12997 return Register();
12998}
12999
13000Register fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13001 if (RetVT.SimpleTy != MVT::v32i8)
13002 return Register();
13003 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13004 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13005 }
13006 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13007 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13008 }
13009 return Register();
13010}
13011
13012Register fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13013 if (RetVT.SimpleTy != MVT::v64i8)
13014 return Register();
13015 if ((Subtarget->hasBWI())) {
13016 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13017 }
13018 return Register();
13019}
13020
13021Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13022 if (RetVT.SimpleTy != MVT::v8i16)
13023 return Register();
13024 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13025 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13026 }
13027 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13028 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13029 }
13030 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13031 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13032 }
13033 return Register();
13034}
13035
13036Register fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13037 if (RetVT.SimpleTy != MVT::v16i16)
13038 return Register();
13039 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13040 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13041 }
13042 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13043 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13044 }
13045 return Register();
13046}
13047
13048Register fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13049 if (RetVT.SimpleTy != MVT::v32i16)
13050 return Register();
13051 if ((Subtarget->hasBWI())) {
13052 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13053 }
13054 return Register();
13055}
13056
13057Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13058 switch (VT.SimpleTy) {
13059 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13060 case MVT::v32i8: return fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
13061 case MVT::v64i8: return fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
13062 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13063 case MVT::v16i16: return fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
13064 case MVT::v32i16: return fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
13065 default: return Register();
13066 }
13067}
13068
13069// FastEmit functions for ISD::STRICT_FADD.
13070
13071Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13072 if (RetVT.SimpleTy != MVT::f16)
13073 return Register();
13074 if ((Subtarget->hasFP16())) {
13075 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13076 }
13077 return Register();
13078}
13079
13080Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13081 if (RetVT.SimpleTy != MVT::f32)
13082 return Register();
13083 if ((Subtarget->hasAVX512())) {
13084 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13085 }
13086 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13087 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13088 }
13089 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13090 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13091 }
13092 if ((!Subtarget->hasSSE1())) {
13093 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13094 }
13095 return Register();
13096}
13097
13098Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13099 if (RetVT.SimpleTy != MVT::f64)
13100 return Register();
13101 if ((Subtarget->hasAVX512())) {
13102 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13103 }
13104 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13105 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13106 }
13107 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13108 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13109 }
13110 if ((!Subtarget->hasSSE2())) {
13111 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13112 }
13113 return Register();
13114}
13115
13116Register fastEmit_ISD_STRICT_FADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13117 if (RetVT.SimpleTy != MVT::f80)
13118 return Register();
13119 if ((Subtarget->hasX87())) {
13120 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13121 }
13122 return Register();
13123}
13124
13125Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13126 if (RetVT.SimpleTy != MVT::v8f16)
13127 return Register();
13128 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13129 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13130 }
13131 return Register();
13132}
13133
13134Register fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13135 if (RetVT.SimpleTy != MVT::v16f16)
13136 return Register();
13137 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13138 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13139 }
13140 return Register();
13141}
13142
13143Register fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13144 if (RetVT.SimpleTy != MVT::v32f16)
13145 return Register();
13146 if ((Subtarget->hasFP16())) {
13147 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13148 }
13149 return Register();
13150}
13151
13152Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13153 if (RetVT.SimpleTy != MVT::v4f32)
13154 return Register();
13155 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13156 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13157 }
13158 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13159 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13160 }
13161 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13162 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13163 }
13164 return Register();
13165}
13166
13167Register fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13168 if (RetVT.SimpleTy != MVT::v8f32)
13169 return Register();
13170 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13171 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13172 }
13173 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13174 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13175 }
13176 return Register();
13177}
13178
13179Register fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13180 if (RetVT.SimpleTy != MVT::v16f32)
13181 return Register();
13182 if ((Subtarget->hasAVX512())) {
13183 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13184 }
13185 return Register();
13186}
13187
13188Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13189 if (RetVT.SimpleTy != MVT::v2f64)
13190 return Register();
13191 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13192 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13193 }
13194 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13195 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13196 }
13197 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13198 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13199 }
13200 return Register();
13201}
13202
13203Register fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13204 if (RetVT.SimpleTy != MVT::v4f64)
13205 return Register();
13206 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13207 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13208 }
13209 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13210 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13211 }
13212 return Register();
13213}
13214
13215Register fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13216 if (RetVT.SimpleTy != MVT::v8f64)
13217 return Register();
13218 if ((Subtarget->hasAVX512())) {
13219 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13220 }
13221 return Register();
13222}
13223
13224Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13225 switch (VT.SimpleTy) {
13226 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
13227 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
13228 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
13229 case MVT::f80: return fastEmit_ISD_STRICT_FADD_MVT_f80_rr(RetVT, Op0, Op1);
13230 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
13231 case MVT::v16f16: return fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
13232 case MVT::v32f16: return fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
13233 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
13234 case MVT::v8f32: return fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
13235 case MVT::v16f32: return fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
13236 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
13237 case MVT::v4f64: return fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
13238 case MVT::v8f64: return fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
13239 default: return Register();
13240 }
13241}
13242
13243// FastEmit functions for ISD::STRICT_FDIV.
13244
13245Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13246 if (RetVT.SimpleTy != MVT::f16)
13247 return Register();
13248 if ((Subtarget->hasFP16())) {
13249 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13250 }
13251 return Register();
13252}
13253
13254Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13255 if (RetVT.SimpleTy != MVT::f32)
13256 return Register();
13257 if ((Subtarget->hasAVX512())) {
13258 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13259 }
13260 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13261 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13262 }
13263 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13264 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13265 }
13266 if ((!Subtarget->hasSSE1())) {
13267 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13268 }
13269 return Register();
13270}
13271
13272Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13273 if (RetVT.SimpleTy != MVT::f64)
13274 return Register();
13275 if ((Subtarget->hasAVX512())) {
13276 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13277 }
13278 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13279 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13280 }
13281 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13282 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13283 }
13284 if ((!Subtarget->hasSSE2())) {
13285 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13286 }
13287 return Register();
13288}
13289
13290Register fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13291 if (RetVT.SimpleTy != MVT::f80)
13292 return Register();
13293 if ((Subtarget->hasX87())) {
13294 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13295 }
13296 return Register();
13297}
13298
13299Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13300 if (RetVT.SimpleTy != MVT::v8f16)
13301 return Register();
13302 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13303 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13304 }
13305 return Register();
13306}
13307
13308Register fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13309 if (RetVT.SimpleTy != MVT::v16f16)
13310 return Register();
13311 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13312 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13313 }
13314 return Register();
13315}
13316
13317Register fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13318 if (RetVT.SimpleTy != MVT::v32f16)
13319 return Register();
13320 if ((Subtarget->hasFP16())) {
13321 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13322 }
13323 return Register();
13324}
13325
13326Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13327 if (RetVT.SimpleTy != MVT::v4f32)
13328 return Register();
13329 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13330 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13331 }
13332 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13333 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13334 }
13335 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13336 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13337 }
13338 return Register();
13339}
13340
13341Register fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13342 if (RetVT.SimpleTy != MVT::v8f32)
13343 return Register();
13344 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13345 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13346 }
13347 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13348 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13349 }
13350 return Register();
13351}
13352
13353Register fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13354 if (RetVT.SimpleTy != MVT::v16f32)
13355 return Register();
13356 if ((Subtarget->hasAVX512())) {
13357 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13358 }
13359 return Register();
13360}
13361
13362Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13363 if (RetVT.SimpleTy != MVT::v2f64)
13364 return Register();
13365 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13366 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13367 }
13368 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13369 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13370 }
13371 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13372 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13373 }
13374 return Register();
13375}
13376
13377Register fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13378 if (RetVT.SimpleTy != MVT::v4f64)
13379 return Register();
13380 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13381 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13382 }
13383 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13384 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13385 }
13386 return Register();
13387}
13388
13389Register fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13390 if (RetVT.SimpleTy != MVT::v8f64)
13391 return Register();
13392 if ((Subtarget->hasAVX512())) {
13393 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13394 }
13395 return Register();
13396}
13397
13398Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13399 switch (VT.SimpleTy) {
13400 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
13401 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
13402 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
13403 case MVT::f80: return fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
13404 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
13405 case MVT::v16f16: return fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
13406 case MVT::v32f16: return fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
13407 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
13408 case MVT::v8f32: return fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
13409 case MVT::v16f32: return fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
13410 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
13411 case MVT::v4f64: return fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
13412 case MVT::v8f64: return fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
13413 default: return Register();
13414 }
13415}
13416
13417// FastEmit functions for ISD::STRICT_FMUL.
13418
13419Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13420 if (RetVT.SimpleTy != MVT::f16)
13421 return Register();
13422 if ((Subtarget->hasFP16())) {
13423 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13424 }
13425 return Register();
13426}
13427
13428Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13429 if (RetVT.SimpleTy != MVT::f32)
13430 return Register();
13431 if ((Subtarget->hasAVX512())) {
13432 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13433 }
13434 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13435 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13436 }
13437 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13438 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13439 }
13440 if ((!Subtarget->hasSSE1())) {
13441 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13442 }
13443 return Register();
13444}
13445
13446Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13447 if (RetVT.SimpleTy != MVT::f64)
13448 return Register();
13449 if ((Subtarget->hasAVX512())) {
13450 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13451 }
13452 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13453 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13454 }
13455 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13456 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13457 }
13458 if ((!Subtarget->hasSSE2())) {
13459 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13460 }
13461 return Register();
13462}
13463
13464Register fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13465 if (RetVT.SimpleTy != MVT::f80)
13466 return Register();
13467 if ((Subtarget->hasX87())) {
13468 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13469 }
13470 return Register();
13471}
13472
13473Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13474 if (RetVT.SimpleTy != MVT::v8f16)
13475 return Register();
13476 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13477 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13478 }
13479 return Register();
13480}
13481
13482Register fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13483 if (RetVT.SimpleTy != MVT::v16f16)
13484 return Register();
13485 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13486 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13487 }
13488 return Register();
13489}
13490
13491Register fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13492 if (RetVT.SimpleTy != MVT::v32f16)
13493 return Register();
13494 if ((Subtarget->hasFP16())) {
13495 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13496 }
13497 return Register();
13498}
13499
13500Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13501 if (RetVT.SimpleTy != MVT::v4f32)
13502 return Register();
13503 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13504 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13505 }
13506 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13507 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13508 }
13509 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13510 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13511 }
13512 return Register();
13513}
13514
13515Register fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13516 if (RetVT.SimpleTy != MVT::v8f32)
13517 return Register();
13518 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13519 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13520 }
13521 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13522 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13523 }
13524 return Register();
13525}
13526
13527Register fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13528 if (RetVT.SimpleTy != MVT::v16f32)
13529 return Register();
13530 if ((Subtarget->hasAVX512())) {
13531 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13532 }
13533 return Register();
13534}
13535
13536Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13537 if (RetVT.SimpleTy != MVT::v2f64)
13538 return Register();
13539 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13540 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13541 }
13542 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13543 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13544 }
13545 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13546 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13547 }
13548 return Register();
13549}
13550
13551Register fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13552 if (RetVT.SimpleTy != MVT::v4f64)
13553 return Register();
13554 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13555 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13556 }
13557 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13558 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13559 }
13560 return Register();
13561}
13562
13563Register fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13564 if (RetVT.SimpleTy != MVT::v8f64)
13565 return Register();
13566 if ((Subtarget->hasAVX512())) {
13567 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13568 }
13569 return Register();
13570}
13571
13572Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13573 switch (VT.SimpleTy) {
13574 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13575 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13576 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13577 case MVT::f80: return fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
13578 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13579 case MVT::v16f16: return fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
13580 case MVT::v32f16: return fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
13581 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13582 case MVT::v8f32: return fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
13583 case MVT::v16f32: return fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
13584 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13585 case MVT::v4f64: return fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
13586 case MVT::v8f64: return fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
13587 default: return Register();
13588 }
13589}
13590
13591// FastEmit functions for ISD::STRICT_FSUB.
13592
13593Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13594 if (RetVT.SimpleTy != MVT::f16)
13595 return Register();
13596 if ((Subtarget->hasFP16())) {
13597 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13598 }
13599 return Register();
13600}
13601
13602Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13603 if (RetVT.SimpleTy != MVT::f32)
13604 return Register();
13605 if ((Subtarget->hasAVX512())) {
13606 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13607 }
13608 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13609 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13610 }
13611 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13612 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13613 }
13614 if ((!Subtarget->hasSSE1())) {
13615 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13616 }
13617 return Register();
13618}
13619
13620Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13621 if (RetVT.SimpleTy != MVT::f64)
13622 return Register();
13623 if ((Subtarget->hasAVX512())) {
13624 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13625 }
13626 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13627 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13628 }
13629 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13630 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13631 }
13632 if ((!Subtarget->hasSSE2())) {
13633 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13634 }
13635 return Register();
13636}
13637
13638Register fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13639 if (RetVT.SimpleTy != MVT::f80)
13640 return Register();
13641 if ((Subtarget->hasX87())) {
13642 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13643 }
13644 return Register();
13645}
13646
13647Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13648 if (RetVT.SimpleTy != MVT::v8f16)
13649 return Register();
13650 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13651 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13652 }
13653 return Register();
13654}
13655
13656Register fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13657 if (RetVT.SimpleTy != MVT::v16f16)
13658 return Register();
13659 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13660 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13661 }
13662 return Register();
13663}
13664
13665Register fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13666 if (RetVT.SimpleTy != MVT::v32f16)
13667 return Register();
13668 if ((Subtarget->hasFP16())) {
13669 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13670 }
13671 return Register();
13672}
13673
13674Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13675 if (RetVT.SimpleTy != MVT::v4f32)
13676 return Register();
13677 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13678 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13679 }
13680 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13681 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13682 }
13683 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13684 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13685 }
13686 return Register();
13687}
13688
13689Register fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13690 if (RetVT.SimpleTy != MVT::v8f32)
13691 return Register();
13692 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13693 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13694 }
13695 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13696 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13697 }
13698 return Register();
13699}
13700
13701Register fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13702 if (RetVT.SimpleTy != MVT::v16f32)
13703 return Register();
13704 if ((Subtarget->hasAVX512())) {
13705 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13706 }
13707 return Register();
13708}
13709
13710Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13711 if (RetVT.SimpleTy != MVT::v2f64)
13712 return Register();
13713 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13714 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13715 }
13716 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13717 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13718 }
13719 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13720 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13721 }
13722 return Register();
13723}
13724
13725Register fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13726 if (RetVT.SimpleTy != MVT::v4f64)
13727 return Register();
13728 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13729 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13730 }
13731 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13732 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13733 }
13734 return Register();
13735}
13736
13737Register fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13738 if (RetVT.SimpleTy != MVT::v8f64)
13739 return Register();
13740 if ((Subtarget->hasAVX512())) {
13741 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13742 }
13743 return Register();
13744}
13745
13746Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13747 switch (VT.SimpleTy) {
13748 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13749 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13750 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13751 case MVT::f80: return fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
13752 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13753 case MVT::v16f16: return fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
13754 case MVT::v32f16: return fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
13755 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13756 case MVT::v8f32: return fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
13757 case MVT::v16f32: return fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
13758 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13759 case MVT::v4f64: return fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
13760 case MVT::v8f64: return fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
13761 default: return Register();
13762 }
13763}
13764
13765// FastEmit functions for ISD::SUB.
13766
13767Register fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13768 if (RetVT.SimpleTy != MVT::i8)
13769 return Register();
13770 if ((Subtarget->hasNDD())) {
13771 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
13772 }
13773 if ((!Subtarget->hasNDD())) {
13774 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr, RC: &X86::GR8RegClass, Op0, Op1);
13775 }
13776 return Register();
13777}
13778
13779Register fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
13780 if (RetVT.SimpleTy != MVT::i16)
13781 return Register();
13782 if ((Subtarget->hasNDD())) {
13783 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
13784 }
13785 if ((!Subtarget->hasNDD())) {
13786 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr, RC: &X86::GR16RegClass, Op0, Op1);
13787 }
13788 return Register();
13789}
13790
13791Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13792 if (RetVT.SimpleTy != MVT::i32)
13793 return Register();
13794 if ((Subtarget->hasNDD())) {
13795 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
13796 }
13797 if ((!Subtarget->hasNDD())) {
13798 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr, RC: &X86::GR32RegClass, Op0, Op1);
13799 }
13800 return Register();
13801}
13802
13803Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13804 if (RetVT.SimpleTy != MVT::i64)
13805 return Register();
13806 if ((Subtarget->hasNDD())) {
13807 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
13808 }
13809 if ((!Subtarget->hasNDD())) {
13810 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr, RC: &X86::GR64RegClass, Op0, Op1);
13811 }
13812 return Register();
13813}
13814
13815Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13816 if (RetVT.SimpleTy != MVT::v16i8)
13817 return Register();
13818 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13819 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13820 }
13821 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13822 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
13823 }
13824 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13825 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
13826 }
13827 return Register();
13828}
13829
13830Register fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13831 if (RetVT.SimpleTy != MVT::v32i8)
13832 return Register();
13833 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13834 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13835 }
13836 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13837 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13838 }
13839 return Register();
13840}
13841
13842Register fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13843 if (RetVT.SimpleTy != MVT::v64i8)
13844 return Register();
13845 if ((Subtarget->hasBWI())) {
13846 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13847 }
13848 return Register();
13849}
13850
13851Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13852 if (RetVT.SimpleTy != MVT::v8i16)
13853 return Register();
13854 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13855 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13856 }
13857 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13858 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
13859 }
13860 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13861 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
13862 }
13863 return Register();
13864}
13865
13866Register fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13867 if (RetVT.SimpleTy != MVT::v16i16)
13868 return Register();
13869 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13870 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13871 }
13872 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13873 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13874 }
13875 return Register();
13876}
13877
13878Register fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13879 if (RetVT.SimpleTy != MVT::v32i16)
13880 return Register();
13881 if ((Subtarget->hasBWI())) {
13882 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13883 }
13884 return Register();
13885}
13886
13887Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13888 if (RetVT.SimpleTy != MVT::v4i32)
13889 return Register();
13890 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13891 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13892 }
13893 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13894 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
13895 }
13896 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13897 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
13898 }
13899 return Register();
13900}
13901
13902Register fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
13903 if (RetVT.SimpleTy != MVT::v8i32)
13904 return Register();
13905 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13906 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13907 }
13908 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13909 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13910 }
13911 return Register();
13912}
13913
13914Register fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
13915 if (RetVT.SimpleTy != MVT::v16i32)
13916 return Register();
13917 if ((Subtarget->hasAVX512())) {
13918 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13919 }
13920 return Register();
13921}
13922
13923Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13924 if (RetVT.SimpleTy != MVT::v2i64)
13925 return Register();
13926 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13927 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13928 }
13929 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13930 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
13931 }
13932 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13933 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
13934 }
13935 return Register();
13936}
13937
13938Register fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
13939 if (RetVT.SimpleTy != MVT::v4i64)
13940 return Register();
13941 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13942 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13943 }
13944 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13945 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQYrr, RC: &X86::VR256RegClass, Op0, Op1);
13946 }
13947 return Register();
13948}
13949
13950Register fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
13951 if (RetVT.SimpleTy != MVT::v8i64)
13952 return Register();
13953 if ((Subtarget->hasAVX512())) {
13954 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13955 }
13956 return Register();
13957}
13958
13959Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13960 switch (VT.SimpleTy) {
13961 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op1);
13962 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op1);
13963 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13964 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13965 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13966 case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op1);
13967 case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op1);
13968 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13969 case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op1);
13970 case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op1);
13971 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13972 case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op1);
13973 case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op1);
13974 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13975 case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op1);
13976 case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op1);
13977 default: return Register();
13978 }
13979}
13980
13981// FastEmit functions for ISD::UADDSAT.
13982
13983Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13984 if (RetVT.SimpleTy != MVT::v16i8)
13985 return Register();
13986 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13987 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13988 }
13989 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13990 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13991 }
13992 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13993 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13994 }
13995 return Register();
13996}
13997
13998Register fastEmit_ISD_UADDSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13999 if (RetVT.SimpleTy != MVT::v32i8)
14000 return Register();
14001 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14002 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14003 }
14004 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14005 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14006 }
14007 return Register();
14008}
14009
14010Register fastEmit_ISD_UADDSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14011 if (RetVT.SimpleTy != MVT::v64i8)
14012 return Register();
14013 if ((Subtarget->hasBWI())) {
14014 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14015 }
14016 return Register();
14017}
14018
14019Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14020 if (RetVT.SimpleTy != MVT::v8i16)
14021 return Register();
14022 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14023 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14024 }
14025 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14026 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14027 }
14028 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14029 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14030 }
14031 return Register();
14032}
14033
14034Register fastEmit_ISD_UADDSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14035 if (RetVT.SimpleTy != MVT::v16i16)
14036 return Register();
14037 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14038 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14039 }
14040 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14041 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14042 }
14043 return Register();
14044}
14045
14046Register fastEmit_ISD_UADDSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14047 if (RetVT.SimpleTy != MVT::v32i16)
14048 return Register();
14049 if ((Subtarget->hasBWI())) {
14050 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14051 }
14052 return Register();
14053}
14054
14055Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14056 switch (VT.SimpleTy) {
14057 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14058 case MVT::v32i8: return fastEmit_ISD_UADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
14059 case MVT::v64i8: return fastEmit_ISD_UADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
14060 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14061 case MVT::v16i16: return fastEmit_ISD_UADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
14062 case MVT::v32i16: return fastEmit_ISD_UADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
14063 default: return Register();
14064 }
14065}
14066
14067// FastEmit functions for ISD::UMAX.
14068
14069Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14070 if (RetVT.SimpleTy != MVT::v16i8)
14071 return Register();
14072 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14073 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14074 }
14075 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14076 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14077 }
14078 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14079 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14080 }
14081 return Register();
14082}
14083
14084Register fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14085 if (RetVT.SimpleTy != MVT::v32i8)
14086 return Register();
14087 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14088 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14089 }
14090 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14091 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14092 }
14093 return Register();
14094}
14095
14096Register fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14097 if (RetVT.SimpleTy != MVT::v64i8)
14098 return Register();
14099 if ((Subtarget->hasBWI())) {
14100 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14101 }
14102 return Register();
14103}
14104
14105Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14106 if (RetVT.SimpleTy != MVT::v8i16)
14107 return Register();
14108 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14109 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14110 }
14111 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14112 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14113 }
14114 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14115 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14116 }
14117 return Register();
14118}
14119
14120Register fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14121 if (RetVT.SimpleTy != MVT::v16i16)
14122 return Register();
14123 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14124 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14125 }
14126 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14127 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14128 }
14129 return Register();
14130}
14131
14132Register fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14133 if (RetVT.SimpleTy != MVT::v32i16)
14134 return Register();
14135 if ((Subtarget->hasBWI())) {
14136 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14137 }
14138 return Register();
14139}
14140
14141Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14142 if (RetVT.SimpleTy != MVT::v4i32)
14143 return Register();
14144 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14145 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14146 }
14147 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14148 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14149 }
14150 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14151 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14152 }
14153 return Register();
14154}
14155
14156Register fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14157 if (RetVT.SimpleTy != MVT::v8i32)
14158 return Register();
14159 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14160 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14161 }
14162 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14163 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14164 }
14165 return Register();
14166}
14167
14168Register fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14169 if (RetVT.SimpleTy != MVT::v16i32)
14170 return Register();
14171 if ((Subtarget->hasAVX512())) {
14172 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14173 }
14174 return Register();
14175}
14176
14177Register fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14178 if (RetVT.SimpleTy != MVT::v2i64)
14179 return Register();
14180 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14181 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14182 }
14183 return Register();
14184}
14185
14186Register fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14187 if (RetVT.SimpleTy != MVT::v4i64)
14188 return Register();
14189 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14190 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14191 }
14192 return Register();
14193}
14194
14195Register fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14196 if (RetVT.SimpleTy != MVT::v8i64)
14197 return Register();
14198 if ((Subtarget->hasAVX512())) {
14199 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14200 }
14201 return Register();
14202}
14203
14204Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14205 switch (VT.SimpleTy) {
14206 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
14207 case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
14208 case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
14209 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
14210 case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
14211 case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
14212 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
14213 case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
14214 case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
14215 case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
14216 case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
14217 case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
14218 default: return Register();
14219 }
14220}
14221
14222// FastEmit functions for ISD::UMIN.
14223
14224Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14225 if (RetVT.SimpleTy != MVT::v16i8)
14226 return Register();
14227 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14228 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14229 }
14230 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14231 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14232 }
14233 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14234 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14235 }
14236 return Register();
14237}
14238
14239Register fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14240 if (RetVT.SimpleTy != MVT::v32i8)
14241 return Register();
14242 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14243 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14244 }
14245 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14246 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14247 }
14248 return Register();
14249}
14250
14251Register fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14252 if (RetVT.SimpleTy != MVT::v64i8)
14253 return Register();
14254 if ((Subtarget->hasBWI())) {
14255 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14256 }
14257 return Register();
14258}
14259
14260Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14261 if (RetVT.SimpleTy != MVT::v8i16)
14262 return Register();
14263 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14264 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14265 }
14266 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14267 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14268 }
14269 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14270 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14271 }
14272 return Register();
14273}
14274
14275Register fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14276 if (RetVT.SimpleTy != MVT::v16i16)
14277 return Register();
14278 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14279 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14280 }
14281 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14282 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14283 }
14284 return Register();
14285}
14286
14287Register fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14288 if (RetVT.SimpleTy != MVT::v32i16)
14289 return Register();
14290 if ((Subtarget->hasBWI())) {
14291 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14292 }
14293 return Register();
14294}
14295
14296Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14297 if (RetVT.SimpleTy != MVT::v4i32)
14298 return Register();
14299 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14300 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14301 }
14302 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14303 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14304 }
14305 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14306 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14307 }
14308 return Register();
14309}
14310
14311Register fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14312 if (RetVT.SimpleTy != MVT::v8i32)
14313 return Register();
14314 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14315 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14316 }
14317 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14318 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14319 }
14320 return Register();
14321}
14322
14323Register fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14324 if (RetVT.SimpleTy != MVT::v16i32)
14325 return Register();
14326 if ((Subtarget->hasAVX512())) {
14327 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14328 }
14329 return Register();
14330}
14331
14332Register fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14333 if (RetVT.SimpleTy != MVT::v2i64)
14334 return Register();
14335 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14336 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14337 }
14338 return Register();
14339}
14340
14341Register fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14342 if (RetVT.SimpleTy != MVT::v4i64)
14343 return Register();
14344 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14345 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14346 }
14347 return Register();
14348}
14349
14350Register fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14351 if (RetVT.SimpleTy != MVT::v8i64)
14352 return Register();
14353 if ((Subtarget->hasAVX512())) {
14354 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14355 }
14356 return Register();
14357}
14358
14359Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14360 switch (VT.SimpleTy) {
14361 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
14362 case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
14363 case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
14364 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
14365 case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
14366 case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
14367 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
14368 case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
14369 case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
14370 case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
14371 case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
14372 case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
14373 default: return Register();
14374 }
14375}
14376
14377// FastEmit functions for ISD::USUBSAT.
14378
14379Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14380 if (RetVT.SimpleTy != MVT::v16i8)
14381 return Register();
14382 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14383 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14384 }
14385 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14386 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14387 }
14388 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14389 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14390 }
14391 return Register();
14392}
14393
14394Register fastEmit_ISD_USUBSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14395 if (RetVT.SimpleTy != MVT::v32i8)
14396 return Register();
14397 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14398 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14399 }
14400 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14401 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14402 }
14403 return Register();
14404}
14405
14406Register fastEmit_ISD_USUBSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14407 if (RetVT.SimpleTy != MVT::v64i8)
14408 return Register();
14409 if ((Subtarget->hasBWI())) {
14410 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14411 }
14412 return Register();
14413}
14414
14415Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14416 if (RetVT.SimpleTy != MVT::v8i16)
14417 return Register();
14418 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14419 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14420 }
14421 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14422 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14423 }
14424 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14425 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14426 }
14427 return Register();
14428}
14429
14430Register fastEmit_ISD_USUBSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14431 if (RetVT.SimpleTy != MVT::v16i16)
14432 return Register();
14433 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14434 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14435 }
14436 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14437 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14438 }
14439 return Register();
14440}
14441
14442Register fastEmit_ISD_USUBSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14443 if (RetVT.SimpleTy != MVT::v32i16)
14444 return Register();
14445 if ((Subtarget->hasBWI())) {
14446 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14447 }
14448 return Register();
14449}
14450
14451Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14452 switch (VT.SimpleTy) {
14453 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14454 case MVT::v32i8: return fastEmit_ISD_USUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
14455 case MVT::v64i8: return fastEmit_ISD_USUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
14456 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14457 case MVT::v16i16: return fastEmit_ISD_USUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
14458 case MVT::v32i16: return fastEmit_ISD_USUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
14459 default: return Register();
14460 }
14461}
14462
14463// FastEmit functions for ISD::XOR.
14464
14465Register fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
14466 if (RetVT.SimpleTy != MVT::i8)
14467 return Register();
14468 if ((Subtarget->hasNDD())) {
14469 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
14470 }
14471 if ((!Subtarget->hasNDD())) {
14472 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr, RC: &X86::GR8RegClass, Op0, Op1);
14473 }
14474 return Register();
14475}
14476
14477Register fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
14478 if (RetVT.SimpleTy != MVT::i16)
14479 return Register();
14480 if ((Subtarget->hasNDD())) {
14481 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
14482 }
14483 if ((!Subtarget->hasNDD())) {
14484 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr, RC: &X86::GR16RegClass, Op0, Op1);
14485 }
14486 return Register();
14487}
14488
14489Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14490 if (RetVT.SimpleTy != MVT::i32)
14491 return Register();
14492 if ((Subtarget->hasNDD())) {
14493 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
14494 }
14495 if ((!Subtarget->hasNDD())) {
14496 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr, RC: &X86::GR32RegClass, Op0, Op1);
14497 }
14498 return Register();
14499}
14500
14501Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14502 if (RetVT.SimpleTy != MVT::i64)
14503 return Register();
14504 if ((Subtarget->hasNDD())) {
14505 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
14506 }
14507 if ((!Subtarget->hasNDD())) {
14508 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr, RC: &X86::GR64RegClass, Op0, Op1);
14509 }
14510 return Register();
14511}
14512
14513Register fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
14514 if (RetVT.SimpleTy != MVT::v8i1)
14515 return Register();
14516 if ((Subtarget->hasDQI())) {
14517 return fastEmitInst_rr(MachineInstOpcode: X86::KXORBkk, RC: &X86::VK8RegClass, Op0, Op1);
14518 }
14519 return Register();
14520}
14521
14522Register fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
14523 if (RetVT.SimpleTy != MVT::v16i1)
14524 return Register();
14525 if ((Subtarget->hasAVX512())) {
14526 return fastEmitInst_rr(MachineInstOpcode: X86::KXORWkk, RC: &X86::VK16RegClass, Op0, Op1);
14527 }
14528 return Register();
14529}
14530
14531Register fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
14532 if (RetVT.SimpleTy != MVT::v32i1)
14533 return Register();
14534 if ((Subtarget->hasBWI())) {
14535 return fastEmitInst_rr(MachineInstOpcode: X86::KXORDkk, RC: &X86::VK32RegClass, Op0, Op1);
14536 }
14537 return Register();
14538}
14539
14540Register fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
14541 if (RetVT.SimpleTy != MVT::v64i1)
14542 return Register();
14543 if ((Subtarget->hasBWI())) {
14544 return fastEmitInst_rr(MachineInstOpcode: X86::KXORQkk, RC: &X86::VK64RegClass, Op0, Op1);
14545 }
14546 return Register();
14547}
14548
14549Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14550 if (RetVT.SimpleTy != MVT::v16i8)
14551 return Register();
14552 if ((Subtarget->hasVLX())) {
14553 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14554 }
14555 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14556 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14557 }
14558 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14559 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14560 }
14561 return Register();
14562}
14563
14564Register fastEmit_ISD_XOR_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14565 if (RetVT.SimpleTy != MVT::v32i8)
14566 return Register();
14567 if ((Subtarget->hasVLX())) {
14568 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14569 }
14570 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14571 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14572 }
14573 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14574 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14575 }
14576 return Register();
14577}
14578
14579Register fastEmit_ISD_XOR_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14580 if (RetVT.SimpleTy != MVT::v64i8)
14581 return Register();
14582 if ((Subtarget->hasAVX512())) {
14583 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14584 }
14585 return Register();
14586}
14587
14588Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14589 if (RetVT.SimpleTy != MVT::v8i16)
14590 return Register();
14591 if ((Subtarget->hasVLX())) {
14592 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14593 }
14594 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14595 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14596 }
14597 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14598 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14599 }
14600 return Register();
14601}
14602
14603Register fastEmit_ISD_XOR_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14604 if (RetVT.SimpleTy != MVT::v16i16)
14605 return Register();
14606 if ((Subtarget->hasVLX())) {
14607 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14608 }
14609 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14610 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14611 }
14612 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14613 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14614 }
14615 return Register();
14616}
14617
14618Register fastEmit_ISD_XOR_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14619 if (RetVT.SimpleTy != MVT::v32i16)
14620 return Register();
14621 if ((Subtarget->hasAVX512())) {
14622 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14623 }
14624 return Register();
14625}
14626
14627Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14628 if (RetVT.SimpleTy != MVT::v4i32)
14629 return Register();
14630 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14631 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14632 }
14633 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14634 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14635 }
14636 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14637 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14638 }
14639 return Register();
14640}
14641
14642Register fastEmit_ISD_XOR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14643 if (RetVT.SimpleTy != MVT::v8i32)
14644 return Register();
14645 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14646 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14647 }
14648 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14649 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14650 }
14651 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14652 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14653 }
14654 return Register();
14655}
14656
14657Register fastEmit_ISD_XOR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14658 if (RetVT.SimpleTy != MVT::v16i32)
14659 return Register();
14660 if ((Subtarget->hasAVX512())) {
14661 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14662 }
14663 return Register();
14664}
14665
14666Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14667 if (RetVT.SimpleTy != MVT::v2i64)
14668 return Register();
14669 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14670 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14671 }
14672 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14673 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14674 }
14675 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14676 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
14677 }
14678 return Register();
14679}
14680
14681Register fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14682 if (RetVT.SimpleTy != MVT::v4i64)
14683 return Register();
14684 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14685 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14686 }
14687 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14688 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14689 }
14690 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14691 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
14692 }
14693 return Register();
14694}
14695
14696Register fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14697 if (RetVT.SimpleTy != MVT::v8i64)
14698 return Register();
14699 if ((Subtarget->hasAVX512())) {
14700 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14701 }
14702 return Register();
14703}
14704
14705Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14706 switch (VT.SimpleTy) {
14707 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op1);
14708 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op1);
14709 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14710 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14711 case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op1);
14712 case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op1);
14713 case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op1);
14714 case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op1);
14715 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14716 case MVT::v32i8: return fastEmit_ISD_XOR_MVT_v32i8_rr(RetVT, Op0, Op1);
14717 case MVT::v64i8: return fastEmit_ISD_XOR_MVT_v64i8_rr(RetVT, Op0, Op1);
14718 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14719 case MVT::v16i16: return fastEmit_ISD_XOR_MVT_v16i16_rr(RetVT, Op0, Op1);
14720 case MVT::v32i16: return fastEmit_ISD_XOR_MVT_v32i16_rr(RetVT, Op0, Op1);
14721 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14722 case MVT::v8i32: return fastEmit_ISD_XOR_MVT_v8i32_rr(RetVT, Op0, Op1);
14723 case MVT::v16i32: return fastEmit_ISD_XOR_MVT_v16i32_rr(RetVT, Op0, Op1);
14724 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14725 case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op1);
14726 case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op1);
14727 default: return Register();
14728 }
14729}
14730
14731// FastEmit functions for X86ISD::ADDSUB.
14732
14733Register fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
14734 if (RetVT.SimpleTy != MVT::v4f32)
14735 return Register();
14736 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
14737 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14738 }
14739 if ((Subtarget->hasAVX())) {
14740 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14741 }
14742 return Register();
14743}
14744
14745Register fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
14746 if (RetVT.SimpleTy != MVT::v8f32)
14747 return Register();
14748 if ((Subtarget->hasAVX())) {
14749 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14750 }
14751 return Register();
14752}
14753
14754Register fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14755 if (RetVT.SimpleTy != MVT::v2f64)
14756 return Register();
14757 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
14758 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14759 }
14760 if ((Subtarget->hasAVX())) {
14761 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14762 }
14763 return Register();
14764}
14765
14766Register fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14767 if (RetVT.SimpleTy != MVT::v4f64)
14768 return Register();
14769 if ((Subtarget->hasAVX())) {
14770 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14771 }
14772 return Register();
14773}
14774
14775Register fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14776 switch (VT.SimpleTy) {
14777 case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
14778 case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
14779 case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
14780 case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
14781 default: return Register();
14782 }
14783}
14784
14785// FastEmit functions for X86ISD::ANDNP.
14786
14787Register fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14788 if (RetVT.SimpleTy != MVT::v16i8)
14789 return Register();
14790 if ((Subtarget->hasVLX())) {
14791 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14792 }
14793 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14794 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14795 }
14796 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14797 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14798 }
14799 return Register();
14800}
14801
14802Register fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14803 if (RetVT.SimpleTy != MVT::v32i8)
14804 return Register();
14805 if ((Subtarget->hasVLX())) {
14806 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14807 }
14808 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14809 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14810 }
14811 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14812 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14813 }
14814 return Register();
14815}
14816
14817Register fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14818 if (RetVT.SimpleTy != MVT::v64i8)
14819 return Register();
14820 if ((Subtarget->hasAVX512())) {
14821 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14822 }
14823 return Register();
14824}
14825
14826Register fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14827 if (RetVT.SimpleTy != MVT::v8i16)
14828 return Register();
14829 if ((Subtarget->hasVLX())) {
14830 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14831 }
14832 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14833 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14834 }
14835 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14836 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14837 }
14838 return Register();
14839}
14840
14841Register fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14842 if (RetVT.SimpleTy != MVT::v16i16)
14843 return Register();
14844 if ((Subtarget->hasVLX())) {
14845 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14846 }
14847 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14848 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14849 }
14850 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14851 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14852 }
14853 return Register();
14854}
14855
14856Register fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14857 if (RetVT.SimpleTy != MVT::v32i16)
14858 return Register();
14859 if ((Subtarget->hasAVX512())) {
14860 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14861 }
14862 return Register();
14863}
14864
14865Register fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14866 if (RetVT.SimpleTy != MVT::v4i32)
14867 return Register();
14868 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14869 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14870 }
14871 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14872 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14873 }
14874 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14875 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14876 }
14877 return Register();
14878}
14879
14880Register fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14881 if (RetVT.SimpleTy != MVT::v8i32)
14882 return Register();
14883 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14884 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14885 }
14886 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14887 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14888 }
14889 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14890 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14891 }
14892 return Register();
14893}
14894
14895Register fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14896 if (RetVT.SimpleTy != MVT::v16i32)
14897 return Register();
14898 if ((Subtarget->hasAVX512())) {
14899 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14900 }
14901 return Register();
14902}
14903
14904Register fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14905 if (RetVT.SimpleTy != MVT::v2i64)
14906 return Register();
14907 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14908 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14909 }
14910 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14911 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14912 }
14913 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14914 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
14915 }
14916 return Register();
14917}
14918
14919Register fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14920 if (RetVT.SimpleTy != MVT::v4i64)
14921 return Register();
14922 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14923 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14924 }
14925 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14926 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14927 }
14928 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14929 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
14930 }
14931 return Register();
14932}
14933
14934Register fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14935 if (RetVT.SimpleTy != MVT::v8i64)
14936 return Register();
14937 if ((Subtarget->hasAVX512())) {
14938 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14939 }
14940 return Register();
14941}
14942
14943Register fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14944 switch (VT.SimpleTy) {
14945 case MVT::v16i8: return fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(RetVT, Op0, Op1);
14946 case MVT::v32i8: return fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(RetVT, Op0, Op1);
14947 case MVT::v64i8: return fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(RetVT, Op0, Op1);
14948 case MVT::v8i16: return fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(RetVT, Op0, Op1);
14949 case MVT::v16i16: return fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(RetVT, Op0, Op1);
14950 case MVT::v32i16: return fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(RetVT, Op0, Op1);
14951 case MVT::v4i32: return fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(RetVT, Op0, Op1);
14952 case MVT::v8i32: return fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(RetVT, Op0, Op1);
14953 case MVT::v16i32: return fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(RetVT, Op0, Op1);
14954 case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op1);
14955 case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op1);
14956 case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op1);
14957 default: return Register();
14958 }
14959}
14960
14961// FastEmit functions for X86ISD::BEXTR.
14962
14963Register fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14964 if (RetVT.SimpleTy != MVT::i32)
14965 return Register();
14966 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
14967 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
14968 }
14969 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
14970 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr, RC: &X86::GR32RegClass, Op0, Op1);
14971 }
14972 return Register();
14973}
14974
14975Register fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14976 if (RetVT.SimpleTy != MVT::i64)
14977 return Register();
14978 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
14979 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
14980 }
14981 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
14982 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr, RC: &X86::GR64RegClass, Op0, Op1);
14983 }
14984 return Register();
14985}
14986
14987Register fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14988 switch (VT.SimpleTy) {
14989 case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op1);
14990 case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op1);
14991 default: return Register();
14992 }
14993}
14994
14995// FastEmit functions for X86ISD::BT.
14996
14997Register fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
14998 if (RetVT.SimpleTy != MVT::i32)
14999 return Register();
15000 return fastEmitInst_rr(MachineInstOpcode: X86::BT16rr, RC: &X86::GR16RegClass, Op0, Op1);
15001}
15002
15003Register fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15004 if (RetVT.SimpleTy != MVT::i32)
15005 return Register();
15006 return fastEmitInst_rr(MachineInstOpcode: X86::BT32rr, RC: &X86::GR32RegClass, Op0, Op1);
15007}
15008
15009Register fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15010 if (RetVT.SimpleTy != MVT::i32)
15011 return Register();
15012 return fastEmitInst_rr(MachineInstOpcode: X86::BT64rr, RC: &X86::GR64RegClass, Op0, Op1);
15013}
15014
15015Register fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15016 switch (VT.SimpleTy) {
15017 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op1);
15018 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op1);
15019 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op1);
15020 default: return Register();
15021 }
15022}
15023
15024// FastEmit functions for X86ISD::BZHI.
15025
15026Register fastEmit_X86ISD_BZHI_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15027 if (RetVT.SimpleTy != MVT::i32)
15028 return Register();
15029 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15030 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
15031 }
15032 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15033 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr, RC: &X86::GR32RegClass, Op0, Op1);
15034 }
15035 return Register();
15036}
15037
15038Register fastEmit_X86ISD_BZHI_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15039 if (RetVT.SimpleTy != MVT::i64)
15040 return Register();
15041 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15042 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
15043 }
15044 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15045 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr, RC: &X86::GR64RegClass, Op0, Op1);
15046 }
15047 return Register();
15048}
15049
15050Register fastEmit_X86ISD_BZHI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15051 switch (VT.SimpleTy) {
15052 case MVT::i32: return fastEmit_X86ISD_BZHI_MVT_i32_rr(RetVT, Op0, Op1);
15053 case MVT::i64: return fastEmit_X86ISD_BZHI_MVT_i64_rr(RetVT, Op0, Op1);
15054 default: return Register();
15055 }
15056}
15057
15058// FastEmit functions for X86ISD::CMP.
15059
15060Register fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
15061 if (RetVT.SimpleTy != MVT::i32)
15062 return Register();
15063 return fastEmitInst_rr(MachineInstOpcode: X86::CMP8rr, RC: &X86::GR8RegClass, Op0, Op1);
15064}
15065
15066Register fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
15067 if (RetVT.SimpleTy != MVT::i32)
15068 return Register();
15069 return fastEmitInst_rr(MachineInstOpcode: X86::CMP16rr, RC: &X86::GR16RegClass, Op0, Op1);
15070}
15071
15072Register fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15073 if (RetVT.SimpleTy != MVT::i32)
15074 return Register();
15075 return fastEmitInst_rr(MachineInstOpcode: X86::CMP32rr, RC: &X86::GR32RegClass, Op0, Op1);
15076}
15077
15078Register fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15079 if (RetVT.SimpleTy != MVT::i32)
15080 return Register();
15081 return fastEmitInst_rr(MachineInstOpcode: X86::CMP64rr, RC: &X86::GR64RegClass, Op0, Op1);
15082}
15083
15084Register fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15085 switch (VT.SimpleTy) {
15086 case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op1);
15087 case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op1);
15088 case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
15089 case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op1);
15090 default: return Register();
15091 }
15092}
15093
15094// FastEmit functions for X86ISD::COMI.
15095
15096Register fastEmit_X86ISD_COMI_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15097 if (RetVT.SimpleTy != MVT::i32)
15098 return Register();
15099 if ((Subtarget->hasFP16())) {
15100 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15101 }
15102 return Register();
15103}
15104
15105Register fastEmit_X86ISD_COMI_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
15106 if (RetVT.SimpleTy != MVT::i32)
15107 return Register();
15108 if ((Subtarget->hasAVX10_2())) {
15109 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISBF16Zrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15110 }
15111 return Register();
15112}
15113
15114Register fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15115 if (RetVT.SimpleTy != MVT::i32)
15116 return Register();
15117 if ((Subtarget->hasAVX512())) {
15118 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15119 }
15120 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15121 return fastEmitInst_rr(MachineInstOpcode: X86::COMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15122 }
15123 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15124 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15125 }
15126 return Register();
15127}
15128
15129Register fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15130 if (RetVT.SimpleTy != MVT::i32)
15131 return Register();
15132 if ((Subtarget->hasAVX512())) {
15133 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15134 }
15135 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15136 return fastEmitInst_rr(MachineInstOpcode: X86::COMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15137 }
15138 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15139 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15140 }
15141 return Register();
15142}
15143
15144Register fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15145 switch (VT.SimpleTy) {
15146 case MVT::v8f16: return fastEmit_X86ISD_COMI_MVT_v8f16_rr(RetVT, Op0, Op1);
15147 case MVT::v8bf16: return fastEmit_X86ISD_COMI_MVT_v8bf16_rr(RetVT, Op0, Op1);
15148 case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op1);
15149 case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op1);
15150 default: return Register();
15151 }
15152}
15153
15154// FastEmit functions for X86ISD::COMX.
15155
15156Register fastEmit_X86ISD_COMX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15157 if (RetVT.SimpleTy != MVT::i32)
15158 return Register();
15159 if ((Subtarget->hasAVX10_2())) {
15160 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15161 }
15162 return Register();
15163}
15164
15165Register fastEmit_X86ISD_COMX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15166 if (RetVT.SimpleTy != MVT::i32)
15167 return Register();
15168 if ((Subtarget->hasAVX10_2())) {
15169 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15170 }
15171 return Register();
15172}
15173
15174Register fastEmit_X86ISD_COMX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15175 if (RetVT.SimpleTy != MVT::i32)
15176 return Register();
15177 if ((Subtarget->hasAVX10_2())) {
15178 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15179 }
15180 return Register();
15181}
15182
15183Register fastEmit_X86ISD_COMX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15184 switch (VT.SimpleTy) {
15185 case MVT::v8f16: return fastEmit_X86ISD_COMX_MVT_v8f16_rr(RetVT, Op0, Op1);
15186 case MVT::v4f32: return fastEmit_X86ISD_COMX_MVT_v4f32_rr(RetVT, Op0, Op1);
15187 case MVT::v2f64: return fastEmit_X86ISD_COMX_MVT_v2f64_rr(RetVT, Op0, Op1);
15188 default: return Register();
15189 }
15190}
15191
15192// FastEmit functions for X86ISD::FADDS.
15193
15194Register fastEmit_X86ISD_FADDS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15195 if (RetVT.SimpleTy != MVT::v8f16)
15196 return Register();
15197 if ((Subtarget->hasFP16())) {
15198 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15199 }
15200 return Register();
15201}
15202
15203Register fastEmit_X86ISD_FADDS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15204 if (RetVT.SimpleTy != MVT::v4f32)
15205 return Register();
15206 if ((Subtarget->hasAVX512())) {
15207 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15208 }
15209 return Register();
15210}
15211
15212Register fastEmit_X86ISD_FADDS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15213 if (RetVT.SimpleTy != MVT::v2f64)
15214 return Register();
15215 if ((Subtarget->hasAVX512())) {
15216 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15217 }
15218 return Register();
15219}
15220
15221Register fastEmit_X86ISD_FADDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15222 switch (VT.SimpleTy) {
15223 case MVT::v8f16: return fastEmit_X86ISD_FADDS_MVT_v8f16_rr(RetVT, Op0, Op1);
15224 case MVT::v4f32: return fastEmit_X86ISD_FADDS_MVT_v4f32_rr(RetVT, Op0, Op1);
15225 case MVT::v2f64: return fastEmit_X86ISD_FADDS_MVT_v2f64_rr(RetVT, Op0, Op1);
15226 default: return Register();
15227 }
15228}
15229
15230// FastEmit functions for X86ISD::FAND.
15231
15232Register fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
15233 if (RetVT.SimpleTy != MVT::f128)
15234 return Register();
15235 if ((Subtarget->hasVLX())) {
15236 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15237 }
15238 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15239 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15240 }
15241 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15242 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15243 }
15244 return Register();
15245}
15246
15247Register fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15248 if (RetVT.SimpleTy != MVT::v4f32)
15249 return Register();
15250 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15251}
15252
15253Register fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15254 switch (VT.SimpleTy) {
15255 case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op1);
15256 case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op1);
15257 default: return Register();
15258 }
15259}
15260
15261// FastEmit functions for X86ISD::FANDN.
15262
15263Register fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15264 if (RetVT.SimpleTy != MVT::v4f32)
15265 return Register();
15266 return fastEmitInst_rr(MachineInstOpcode: X86::ANDNPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15267}
15268
15269Register fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15270 switch (VT.SimpleTy) {
15271 case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op1);
15272 default: return Register();
15273 }
15274}
15275
15276// FastEmit functions for X86ISD::FCMP.
15277
15278Register fastEmit_X86ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
15279 if (RetVT.SimpleTy != MVT::i32)
15280 return Register();
15281 if ((Subtarget->hasFP16())) {
15282 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
15283 }
15284 return Register();
15285}
15286
15287Register fastEmit_X86ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
15288 if (RetVT.SimpleTy != MVT::i32)
15289 return Register();
15290 if ((Subtarget->hasAVX512())) {
15291 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
15292 }
15293 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15294 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
15295 }
15296 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15297 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
15298 }
15299 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
15300 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
15301 }
15302 return Register();
15303}
15304
15305Register fastEmit_X86ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
15306 if (RetVT.SimpleTy != MVT::i32)
15307 return Register();
15308 if ((Subtarget->hasAVX512())) {
15309 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
15310 }
15311 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15312 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
15313 }
15314 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15315 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
15316 }
15317 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
15318 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
15319 }
15320 return Register();
15321}
15322
15323Register fastEmit_X86ISD_FCMP_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
15324 if (RetVT.SimpleTy != MVT::i32)
15325 return Register();
15326 if ((Subtarget->canUseCMOV())) {
15327 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
15328 }
15329 return Register();
15330}
15331
15332Register fastEmit_X86ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15333 switch (VT.SimpleTy) {
15334 case MVT::f16: return fastEmit_X86ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
15335 case MVT::f32: return fastEmit_X86ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
15336 case MVT::f64: return fastEmit_X86ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
15337 case MVT::f80: return fastEmit_X86ISD_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
15338 default: return Register();
15339 }
15340}
15341
15342// FastEmit functions for X86ISD::FDIVS.
15343
15344Register fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15345 if (RetVT.SimpleTy != MVT::v8f16)
15346 return Register();
15347 if ((Subtarget->hasFP16())) {
15348 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15349 }
15350 return Register();
15351}
15352
15353Register fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15354 if (RetVT.SimpleTy != MVT::v4f32)
15355 return Register();
15356 if ((Subtarget->hasAVX512())) {
15357 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15358 }
15359 return Register();
15360}
15361
15362Register fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15363 if (RetVT.SimpleTy != MVT::v2f64)
15364 return Register();
15365 if ((Subtarget->hasAVX512())) {
15366 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15367 }
15368 return Register();
15369}
15370
15371Register fastEmit_X86ISD_FDIVS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15372 switch (VT.SimpleTy) {
15373 case MVT::v8f16: return fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(RetVT, Op0, Op1);
15374 case MVT::v4f32: return fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(RetVT, Op0, Op1);
15375 case MVT::v2f64: return fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(RetVT, Op0, Op1);
15376 default: return Register();
15377 }
15378}
15379
15380// FastEmit functions for X86ISD::FGETEXPS.
15381
15382Register fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15383 if (RetVT.SimpleTy != MVT::v8f16)
15384 return Register();
15385 if ((Subtarget->hasFP16())) {
15386 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZr, RC: &X86::VR128XRegClass, Op0, Op1);
15387 }
15388 return Register();
15389}
15390
15391Register fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15392 if (RetVT.SimpleTy != MVT::v4f32)
15393 return Register();
15394 if ((Subtarget->hasAVX512())) {
15395 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZr, RC: &X86::VR128XRegClass, Op0, Op1);
15396 }
15397 return Register();
15398}
15399
15400Register fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15401 if (RetVT.SimpleTy != MVT::v2f64)
15402 return Register();
15403 if ((Subtarget->hasAVX512())) {
15404 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZr, RC: &X86::VR128XRegClass, Op0, Op1);
15405 }
15406 return Register();
15407}
15408
15409Register fastEmit_X86ISD_FGETEXPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15410 switch (VT.SimpleTy) {
15411 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(RetVT, Op0, Op1);
15412 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(RetVT, Op0, Op1);
15413 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(RetVT, Op0, Op1);
15414 default: return Register();
15415 }
15416}
15417
15418// FastEmit functions for X86ISD::FGETEXPS_SAE.
15419
15420Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15421 if (RetVT.SimpleTy != MVT::v8f16)
15422 return Register();
15423 if ((Subtarget->hasFP16())) {
15424 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZrb, RC: &X86::VR128XRegClass, Op0, Op1);
15425 }
15426 return Register();
15427}
15428
15429Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15430 if (RetVT.SimpleTy != MVT::v4f32)
15431 return Register();
15432 if ((Subtarget->hasAVX512())) {
15433 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZrb, RC: &X86::VR128XRegClass, Op0, Op1);
15434 }
15435 return Register();
15436}
15437
15438Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15439 if (RetVT.SimpleTy != MVT::v2f64)
15440 return Register();
15441 if ((Subtarget->hasAVX512())) {
15442 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZrb, RC: &X86::VR128XRegClass, Op0, Op1);
15443 }
15444 return Register();
15445}
15446
15447Register fastEmit_X86ISD_FGETEXPS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15448 switch (VT.SimpleTy) {
15449 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
15450 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
15451 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
15452 default: return Register();
15453 }
15454}
15455
15456// FastEmit functions for X86ISD::FHADD.
15457
15458Register fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15459 if (RetVT.SimpleTy != MVT::v4f32)
15460 return Register();
15461 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15462 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15463 }
15464 if ((Subtarget->hasAVX())) {
15465 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15466 }
15467 return Register();
15468}
15469
15470Register fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15471 if (RetVT.SimpleTy != MVT::v8f32)
15472 return Register();
15473 if ((Subtarget->hasAVX())) {
15474 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15475 }
15476 return Register();
15477}
15478
15479Register fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15480 if (RetVT.SimpleTy != MVT::v2f64)
15481 return Register();
15482 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15483 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15484 }
15485 if ((Subtarget->hasAVX())) {
15486 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15487 }
15488 return Register();
15489}
15490
15491Register fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15492 if (RetVT.SimpleTy != MVT::v4f64)
15493 return Register();
15494 if ((Subtarget->hasAVX())) {
15495 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15496 }
15497 return Register();
15498}
15499
15500Register fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15501 switch (VT.SimpleTy) {
15502 case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op1);
15503 case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op1);
15504 case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op1);
15505 case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op1);
15506 default: return Register();
15507 }
15508}
15509
15510// FastEmit functions for X86ISD::FHSUB.
15511
15512Register fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15513 if (RetVT.SimpleTy != MVT::v4f32)
15514 return Register();
15515 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15516 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15517 }
15518 if ((Subtarget->hasAVX())) {
15519 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15520 }
15521 return Register();
15522}
15523
15524Register fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15525 if (RetVT.SimpleTy != MVT::v8f32)
15526 return Register();
15527 if ((Subtarget->hasAVX())) {
15528 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15529 }
15530 return Register();
15531}
15532
15533Register fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15534 if (RetVT.SimpleTy != MVT::v2f64)
15535 return Register();
15536 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15537 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15538 }
15539 if ((Subtarget->hasAVX())) {
15540 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15541 }
15542 return Register();
15543}
15544
15545Register fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15546 if (RetVT.SimpleTy != MVT::v4f64)
15547 return Register();
15548 if ((Subtarget->hasAVX())) {
15549 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15550 }
15551 return Register();
15552}
15553
15554Register fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15555 switch (VT.SimpleTy) {
15556 case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
15557 case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
15558 case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
15559 case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
15560 default: return Register();
15561 }
15562}
15563
15564// FastEmit functions for X86ISD::FMAX.
15565
15566Register fastEmit_X86ISD_FMAX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
15567 if (RetVT.SimpleTy != MVT::f16)
15568 return Register();
15569 if ((Subtarget->hasFP16())) {
15570 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
15571 }
15572 return Register();
15573}
15574
15575Register fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
15576 if (RetVT.SimpleTy != MVT::f32)
15577 return Register();
15578 if ((Subtarget->hasAVX512())) {
15579 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
15580 }
15581 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15582 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15583 }
15584 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15585 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15586 }
15587 return Register();
15588}
15589
15590Register fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
15591 if (RetVT.SimpleTy != MVT::f64)
15592 return Register();
15593 if ((Subtarget->hasAVX512())) {
15594 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
15595 }
15596 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15597 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15598 }
15599 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15600 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15601 }
15602 return Register();
15603}
15604
15605Register fastEmit_X86ISD_FMAX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15606 if (RetVT.SimpleTy != MVT::v8f16)
15607 return Register();
15608 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15609 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15610 }
15611 return Register();
15612}
15613
15614Register fastEmit_X86ISD_FMAX_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
15615 if (RetVT.SimpleTy != MVT::v16f16)
15616 return Register();
15617 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15618 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15619 }
15620 return Register();
15621}
15622
15623Register fastEmit_X86ISD_FMAX_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
15624 if (RetVT.SimpleTy != MVT::v32f16)
15625 return Register();
15626 if ((Subtarget->hasFP16())) {
15627 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
15628 }
15629 return Register();
15630}
15631
15632Register fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15633 if (RetVT.SimpleTy != MVT::v4f32)
15634 return Register();
15635 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15636 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15637 }
15638 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15639 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15640 }
15641 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15642 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15643 }
15644 return Register();
15645}
15646
15647Register fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15648 if (RetVT.SimpleTy != MVT::v8f32)
15649 return Register();
15650 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15651 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15652 }
15653 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15654 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15655 }
15656 return Register();
15657}
15658
15659Register fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
15660 if (RetVT.SimpleTy != MVT::v16f32)
15661 return Register();
15662 if ((Subtarget->hasAVX512())) {
15663 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
15664 }
15665 return Register();
15666}
15667
15668Register fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15669 if (RetVT.SimpleTy != MVT::v2f64)
15670 return Register();
15671 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15672 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15673 }
15674 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15675 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15676 }
15677 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15678 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15679 }
15680 return Register();
15681}
15682
15683Register fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15684 if (RetVT.SimpleTy != MVT::v4f64)
15685 return Register();
15686 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15687 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15688 }
15689 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15690 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15691 }
15692 return Register();
15693}
15694
15695Register fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
15696 if (RetVT.SimpleTy != MVT::v8f64)
15697 return Register();
15698 if ((Subtarget->hasAVX512())) {
15699 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15700 }
15701 return Register();
15702}
15703
15704Register fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15705 switch (VT.SimpleTy) {
15706 case MVT::f16: return fastEmit_X86ISD_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
15707 case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
15708 case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
15709 case MVT::v8f16: return fastEmit_X86ISD_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
15710 case MVT::v16f16: return fastEmit_X86ISD_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
15711 case MVT::v32f16: return fastEmit_X86ISD_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
15712 case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
15713 case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
15714 case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
15715 case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
15716 case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
15717 case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
15718 default: return Register();
15719 }
15720}
15721
15722// FastEmit functions for X86ISD::FMAXC.
15723
15724Register fastEmit_X86ISD_FMAXC_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
15725 if (RetVT.SimpleTy != MVT::f16)
15726 return Register();
15727 if ((Subtarget->hasAVX512())) {
15728 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
15729 }
15730 return Register();
15731}
15732
15733Register fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
15734 if (RetVT.SimpleTy != MVT::f32)
15735 return Register();
15736 if ((Subtarget->hasAVX512())) {
15737 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
15738 }
15739 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15740 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15741 }
15742 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15743 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
15744 }
15745 return Register();
15746}
15747
15748Register fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
15749 if (RetVT.SimpleTy != MVT::f64)
15750 return Register();
15751 if ((Subtarget->hasAVX512())) {
15752 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
15753 }
15754 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15755 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15756 }
15757 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15758 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
15759 }
15760 return Register();
15761}
15762
15763Register fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15764 if (RetVT.SimpleTy != MVT::v8f16)
15765 return Register();
15766 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15767 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15768 }
15769 return Register();
15770}
15771
15772Register fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
15773 if (RetVT.SimpleTy != MVT::v16f16)
15774 return Register();
15775 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
15776 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15777 }
15778 return Register();
15779}
15780
15781Register fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
15782 if (RetVT.SimpleTy != MVT::v32f16)
15783 return Register();
15784 if ((Subtarget->hasFP16())) {
15785 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
15786 }
15787 return Register();
15788}
15789
15790Register fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15791 if (RetVT.SimpleTy != MVT::v4f32)
15792 return Register();
15793 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15794 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15795 }
15796 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15797 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15798 }
15799 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15800 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15801 }
15802 return Register();
15803}
15804
15805Register fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15806 if (RetVT.SimpleTy != MVT::v8f32)
15807 return Register();
15808 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15809 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15810 }
15811 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15812 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15813 }
15814 return Register();
15815}
15816
15817Register fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
15818 if (RetVT.SimpleTy != MVT::v16f32)
15819 return Register();
15820 if ((Subtarget->hasAVX512())) {
15821 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
15822 }
15823 return Register();
15824}
15825
15826Register fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15827 if (RetVT.SimpleTy != MVT::v2f64)
15828 return Register();
15829 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15830 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15831 }
15832 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15833 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15834 }
15835 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15836 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15837 }
15838 return Register();
15839}
15840
15841Register fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15842 if (RetVT.SimpleTy != MVT::v4f64)
15843 return Register();
15844 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15845 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15846 }
15847 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15848 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15849 }
15850 return Register();
15851}
15852
15853Register fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
15854 if (RetVT.SimpleTy != MVT::v8f64)
15855 return Register();
15856 if ((Subtarget->hasAVX512())) {
15857 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15858 }
15859 return Register();
15860}
15861
15862Register fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15863 switch (VT.SimpleTy) {
15864 case MVT::f16: return fastEmit_X86ISD_FMAXC_MVT_f16_rr(RetVT, Op0, Op1);
15865 case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op1);
15866 case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op1);
15867 case MVT::v8f16: return fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(RetVT, Op0, Op1);
15868 case MVT::v16f16: return fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(RetVT, Op0, Op1);
15869 case MVT::v32f16: return fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(RetVT, Op0, Op1);
15870 case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op1);
15871 case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op1);
15872 case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op1);
15873 case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op1);
15874 case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op1);
15875 case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op1);
15876 default: return Register();
15877 }
15878}
15879
15880// FastEmit functions for X86ISD::FMAXS.
15881
15882Register fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15883 if (RetVT.SimpleTy != MVT::v8f16)
15884 return Register();
15885 if ((Subtarget->hasFP16())) {
15886 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15887 }
15888 return Register();
15889}
15890
15891Register fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15892 if (RetVT.SimpleTy != MVT::v4f32)
15893 return Register();
15894 if ((Subtarget->hasAVX512())) {
15895 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15896 }
15897 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15898 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15899 }
15900 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15901 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15902 }
15903 return Register();
15904}
15905
15906Register fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15907 if (RetVT.SimpleTy != MVT::v2f64)
15908 return Register();
15909 if ((Subtarget->hasAVX512())) {
15910 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15911 }
15912 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15913 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15914 }
15915 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15916 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15917 }
15918 return Register();
15919}
15920
15921Register fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15922 switch (VT.SimpleTy) {
15923 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(RetVT, Op0, Op1);
15924 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op1);
15925 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op1);
15926 default: return Register();
15927 }
15928}
15929
15930// FastEmit functions for X86ISD::FMAXS_SAE.
15931
15932Register fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15933 if (RetVT.SimpleTy != MVT::v8f16)
15934 return Register();
15935 if ((Subtarget->hasFP16())) {
15936 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15937 }
15938 return Register();
15939}
15940
15941Register fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15942 if (RetVT.SimpleTy != MVT::v4f32)
15943 return Register();
15944 if ((Subtarget->hasAVX512())) {
15945 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15946 }
15947 return Register();
15948}
15949
15950Register fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15951 if (RetVT.SimpleTy != MVT::v2f64)
15952 return Register();
15953 if ((Subtarget->hasAVX512())) {
15954 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15955 }
15956 return Register();
15957}
15958
15959Register fastEmit_X86ISD_FMAXS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15960 switch (VT.SimpleTy) {
15961 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
15962 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
15963 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
15964 default: return Register();
15965 }
15966}
15967
15968// FastEmit functions for X86ISD::FMAX_SAE.
15969
15970Register fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
15971 if (RetVT.SimpleTy != MVT::v32f16)
15972 return Register();
15973 if ((Subtarget->hasFP16())) {
15974 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
15975 }
15976 return Register();
15977}
15978
15979Register fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
15980 if (RetVT.SimpleTy != MVT::v16f32)
15981 return Register();
15982 if ((Subtarget->hasAVX512())) {
15983 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
15984 }
15985 return Register();
15986}
15987
15988Register fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
15989 if (RetVT.SimpleTy != MVT::v8f64)
15990 return Register();
15991 if ((Subtarget->hasAVX512())) {
15992 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
15993 }
15994 return Register();
15995}
15996
15997Register fastEmit_X86ISD_FMAX_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15998 switch (VT.SimpleTy) {
15999 case MVT::v32f16: return fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
16000 case MVT::v16f32: return fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
16001 case MVT::v8f64: return fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
16002 default: return Register();
16003 }
16004}
16005
16006// FastEmit functions for X86ISD::FMIN.
16007
16008Register fastEmit_X86ISD_FMIN_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16009 if (RetVT.SimpleTy != MVT::f16)
16010 return Register();
16011 if ((Subtarget->hasFP16())) {
16012 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16013 }
16014 return Register();
16015}
16016
16017Register fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16018 if (RetVT.SimpleTy != MVT::f32)
16019 return Register();
16020 if ((Subtarget->hasAVX512())) {
16021 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16022 }
16023 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16024 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16025 }
16026 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16027 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16028 }
16029 return Register();
16030}
16031
16032Register fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16033 if (RetVT.SimpleTy != MVT::f64)
16034 return Register();
16035 if ((Subtarget->hasAVX512())) {
16036 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16037 }
16038 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16039 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16040 }
16041 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16042 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16043 }
16044 return Register();
16045}
16046
16047Register fastEmit_X86ISD_FMIN_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16048 if (RetVT.SimpleTy != MVT::v8f16)
16049 return Register();
16050 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16051 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16052 }
16053 return Register();
16054}
16055
16056Register fastEmit_X86ISD_FMIN_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
16057 if (RetVT.SimpleTy != MVT::v16f16)
16058 return Register();
16059 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16060 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16061 }
16062 return Register();
16063}
16064
16065Register fastEmit_X86ISD_FMIN_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16066 if (RetVT.SimpleTy != MVT::v32f16)
16067 return Register();
16068 if ((Subtarget->hasFP16())) {
16069 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16070 }
16071 return Register();
16072}
16073
16074Register fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16075 if (RetVT.SimpleTy != MVT::v4f32)
16076 return Register();
16077 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16078 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16079 }
16080 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16081 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16082 }
16083 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16084 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16085 }
16086 return Register();
16087}
16088
16089Register fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16090 if (RetVT.SimpleTy != MVT::v8f32)
16091 return Register();
16092 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16093 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16094 }
16095 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16096 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16097 }
16098 return Register();
16099}
16100
16101Register fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16102 if (RetVT.SimpleTy != MVT::v16f32)
16103 return Register();
16104 if ((Subtarget->hasAVX512())) {
16105 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16106 }
16107 return Register();
16108}
16109
16110Register fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16111 if (RetVT.SimpleTy != MVT::v2f64)
16112 return Register();
16113 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16114 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16115 }
16116 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16117 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16118 }
16119 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16120 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16121 }
16122 return Register();
16123}
16124
16125Register fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16126 if (RetVT.SimpleTy != MVT::v4f64)
16127 return Register();
16128 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16129 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16130 }
16131 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16132 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16133 }
16134 return Register();
16135}
16136
16137Register fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16138 if (RetVT.SimpleTy != MVT::v8f64)
16139 return Register();
16140 if ((Subtarget->hasAVX512())) {
16141 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16142 }
16143 return Register();
16144}
16145
16146Register fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16147 switch (VT.SimpleTy) {
16148 case MVT::f16: return fastEmit_X86ISD_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
16149 case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
16150 case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
16151 case MVT::v8f16: return fastEmit_X86ISD_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
16152 case MVT::v16f16: return fastEmit_X86ISD_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
16153 case MVT::v32f16: return fastEmit_X86ISD_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
16154 case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
16155 case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
16156 case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
16157 case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
16158 case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
16159 case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
16160 default: return Register();
16161 }
16162}
16163
16164// FastEmit functions for X86ISD::FMINC.
16165
16166Register fastEmit_X86ISD_FMINC_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16167 if (RetVT.SimpleTy != MVT::f16)
16168 return Register();
16169 if ((Subtarget->hasAVX512())) {
16170 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16171 }
16172 return Register();
16173}
16174
16175Register fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16176 if (RetVT.SimpleTy != MVT::f32)
16177 return Register();
16178 if ((Subtarget->hasAVX512())) {
16179 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16180 }
16181 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16182 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16183 }
16184 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16185 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16186 }
16187 return Register();
16188}
16189
16190Register fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16191 if (RetVT.SimpleTy != MVT::f64)
16192 return Register();
16193 if ((Subtarget->hasAVX512())) {
16194 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16195 }
16196 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16197 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16198 }
16199 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16200 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16201 }
16202 return Register();
16203}
16204
16205Register fastEmit_X86ISD_FMINC_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16206 if (RetVT.SimpleTy != MVT::v8f16)
16207 return Register();
16208 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16209 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16210 }
16211 return Register();
16212}
16213
16214Register fastEmit_X86ISD_FMINC_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
16215 if (RetVT.SimpleTy != MVT::v16f16)
16216 return Register();
16217 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16218 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16219 }
16220 return Register();
16221}
16222
16223Register fastEmit_X86ISD_FMINC_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16224 if (RetVT.SimpleTy != MVT::v32f16)
16225 return Register();
16226 if ((Subtarget->hasFP16())) {
16227 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16228 }
16229 return Register();
16230}
16231
16232Register fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16233 if (RetVT.SimpleTy != MVT::v4f32)
16234 return Register();
16235 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16236 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16237 }
16238 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16239 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16240 }
16241 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16242 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16243 }
16244 return Register();
16245}
16246
16247Register fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16248 if (RetVT.SimpleTy != MVT::v8f32)
16249 return Register();
16250 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16251 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16252 }
16253 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16254 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16255 }
16256 return Register();
16257}
16258
16259Register fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16260 if (RetVT.SimpleTy != MVT::v16f32)
16261 return Register();
16262 if ((Subtarget->hasAVX512())) {
16263 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16264 }
16265 return Register();
16266}
16267
16268Register fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16269 if (RetVT.SimpleTy != MVT::v2f64)
16270 return Register();
16271 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16272 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16273 }
16274 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16275 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16276 }
16277 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16278 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16279 }
16280 return Register();
16281}
16282
16283Register fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16284 if (RetVT.SimpleTy != MVT::v4f64)
16285 return Register();
16286 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16287 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16288 }
16289 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16290 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16291 }
16292 return Register();
16293}
16294
16295Register fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16296 if (RetVT.SimpleTy != MVT::v8f64)
16297 return Register();
16298 if ((Subtarget->hasAVX512())) {
16299 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16300 }
16301 return Register();
16302}
16303
16304Register fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16305 switch (VT.SimpleTy) {
16306 case MVT::f16: return fastEmit_X86ISD_FMINC_MVT_f16_rr(RetVT, Op0, Op1);
16307 case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op1);
16308 case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op1);
16309 case MVT::v8f16: return fastEmit_X86ISD_FMINC_MVT_v8f16_rr(RetVT, Op0, Op1);
16310 case MVT::v16f16: return fastEmit_X86ISD_FMINC_MVT_v16f16_rr(RetVT, Op0, Op1);
16311 case MVT::v32f16: return fastEmit_X86ISD_FMINC_MVT_v32f16_rr(RetVT, Op0, Op1);
16312 case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op1);
16313 case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op1);
16314 case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op1);
16315 case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op1);
16316 case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op1);
16317 case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op1);
16318 default: return Register();
16319 }
16320}
16321
16322// FastEmit functions for X86ISD::FMINS.
16323
16324Register fastEmit_X86ISD_FMINS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16325 if (RetVT.SimpleTy != MVT::v8f16)
16326 return Register();
16327 if ((Subtarget->hasFP16())) {
16328 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16329 }
16330 return Register();
16331}
16332
16333Register fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16334 if (RetVT.SimpleTy != MVT::v4f32)
16335 return Register();
16336 if ((Subtarget->hasAVX512())) {
16337 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16338 }
16339 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16340 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16341 }
16342 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16343 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16344 }
16345 return Register();
16346}
16347
16348Register fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16349 if (RetVT.SimpleTy != MVT::v2f64)
16350 return Register();
16351 if ((Subtarget->hasAVX512())) {
16352 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16353 }
16354 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16355 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16356 }
16357 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16358 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16359 }
16360 return Register();
16361}
16362
16363Register fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16364 switch (VT.SimpleTy) {
16365 case MVT::v8f16: return fastEmit_X86ISD_FMINS_MVT_v8f16_rr(RetVT, Op0, Op1);
16366 case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op1);
16367 case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op1);
16368 default: return Register();
16369 }
16370}
16371
16372// FastEmit functions for X86ISD::FMINS_SAE.
16373
16374Register fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16375 if (RetVT.SimpleTy != MVT::v8f16)
16376 return Register();
16377 if ((Subtarget->hasFP16())) {
16378 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16379 }
16380 return Register();
16381}
16382
16383Register fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16384 if (RetVT.SimpleTy != MVT::v4f32)
16385 return Register();
16386 if ((Subtarget->hasAVX512())) {
16387 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16388 }
16389 return Register();
16390}
16391
16392Register fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16393 if (RetVT.SimpleTy != MVT::v2f64)
16394 return Register();
16395 if ((Subtarget->hasAVX512())) {
16396 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16397 }
16398 return Register();
16399}
16400
16401Register fastEmit_X86ISD_FMINS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16402 switch (VT.SimpleTy) {
16403 case MVT::v8f16: return fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
16404 case MVT::v4f32: return fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16405 case MVT::v2f64: return fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16406 default: return Register();
16407 }
16408}
16409
16410// FastEmit functions for X86ISD::FMIN_SAE.
16411
16412Register fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16413 if (RetVT.SimpleTy != MVT::v32f16)
16414 return Register();
16415 if ((Subtarget->hasFP16())) {
16416 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16417 }
16418 return Register();
16419}
16420
16421Register fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16422 if (RetVT.SimpleTy != MVT::v16f32)
16423 return Register();
16424 if ((Subtarget->hasAVX512())) {
16425 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16426 }
16427 return Register();
16428}
16429
16430Register fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16431 if (RetVT.SimpleTy != MVT::v8f64)
16432 return Register();
16433 if ((Subtarget->hasAVX512())) {
16434 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16435 }
16436 return Register();
16437}
16438
16439Register fastEmit_X86ISD_FMIN_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16440 switch (VT.SimpleTy) {
16441 case MVT::v32f16: return fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
16442 case MVT::v16f32: return fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
16443 case MVT::v8f64: return fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
16444 default: return Register();
16445 }
16446}
16447
16448// FastEmit functions for X86ISD::FMULS.
16449
16450Register fastEmit_X86ISD_FMULS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16451 if (RetVT.SimpleTy != MVT::v8f16)
16452 return Register();
16453 if ((Subtarget->hasFP16())) {
16454 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16455 }
16456 return Register();
16457}
16458
16459Register fastEmit_X86ISD_FMULS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16460 if (RetVT.SimpleTy != MVT::v4f32)
16461 return Register();
16462 if ((Subtarget->hasAVX512())) {
16463 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16464 }
16465 return Register();
16466}
16467
16468Register fastEmit_X86ISD_FMULS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16469 if (RetVT.SimpleTy != MVT::v2f64)
16470 return Register();
16471 if ((Subtarget->hasAVX512())) {
16472 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16473 }
16474 return Register();
16475}
16476
16477Register fastEmit_X86ISD_FMULS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16478 switch (VT.SimpleTy) {
16479 case MVT::v8f16: return fastEmit_X86ISD_FMULS_MVT_v8f16_rr(RetVT, Op0, Op1);
16480 case MVT::v4f32: return fastEmit_X86ISD_FMULS_MVT_v4f32_rr(RetVT, Op0, Op1);
16481 case MVT::v2f64: return fastEmit_X86ISD_FMULS_MVT_v2f64_rr(RetVT, Op0, Op1);
16482 default: return Register();
16483 }
16484}
16485
16486// FastEmit functions for X86ISD::FOR.
16487
16488Register fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
16489 if (RetVT.SimpleTy != MVT::f128)
16490 return Register();
16491 if ((Subtarget->hasVLX())) {
16492 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16493 }
16494 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16495 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16496 }
16497 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16498 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16499 }
16500 return Register();
16501}
16502
16503Register fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16504 if (RetVT.SimpleTy != MVT::v4f32)
16505 return Register();
16506 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16507}
16508
16509Register fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16510 switch (VT.SimpleTy) {
16511 case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op1);
16512 case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op1);
16513 default: return Register();
16514 }
16515}
16516
16517// FastEmit functions for X86ISD::FP80_ADD.
16518
16519Register fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
16520 if (RetVT.SimpleTy != MVT::f80)
16521 return Register();
16522 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
16523}
16524
16525Register fastEmit_X86ISD_FP80_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16526 switch (VT.SimpleTy) {
16527 case MVT::f80: return fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
16528 default: return Register();
16529 }
16530}
16531
16532// FastEmit functions for X86ISD::FSQRTS.
16533
16534Register fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16535 if (RetVT.SimpleTy != MVT::v8f16)
16536 return Register();
16537 if ((Subtarget->hasFP16())) {
16538 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSHZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16539 }
16540 return Register();
16541}
16542
16543Register fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16544 if (RetVT.SimpleTy != MVT::v4f32)
16545 return Register();
16546 if ((Subtarget->hasAVX512())) {
16547 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSSZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16548 }
16549 return Register();
16550}
16551
16552Register fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16553 if (RetVT.SimpleTy != MVT::v2f64)
16554 return Register();
16555 if ((Subtarget->hasAVX512())) {
16556 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSDZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16557 }
16558 return Register();
16559}
16560
16561Register fastEmit_X86ISD_FSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16562 switch (VT.SimpleTy) {
16563 case MVT::v8f16: return fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(RetVT, Op0, Op1);
16564 case MVT::v4f32: return fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
16565 case MVT::v2f64: return fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
16566 default: return Register();
16567 }
16568}
16569
16570// FastEmit functions for X86ISD::FSUBS.
16571
16572Register fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16573 if (RetVT.SimpleTy != MVT::v8f16)
16574 return Register();
16575 if ((Subtarget->hasFP16())) {
16576 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16577 }
16578 return Register();
16579}
16580
16581Register fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16582 if (RetVT.SimpleTy != MVT::v4f32)
16583 return Register();
16584 if ((Subtarget->hasAVX512())) {
16585 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16586 }
16587 return Register();
16588}
16589
16590Register fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16591 if (RetVT.SimpleTy != MVT::v2f64)
16592 return Register();
16593 if ((Subtarget->hasAVX512())) {
16594 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16595 }
16596 return Register();
16597}
16598
16599Register fastEmit_X86ISD_FSUBS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16600 switch (VT.SimpleTy) {
16601 case MVT::v8f16: return fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(RetVT, Op0, Op1);
16602 case MVT::v4f32: return fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(RetVT, Op0, Op1);
16603 case MVT::v2f64: return fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(RetVT, Op0, Op1);
16604 default: return Register();
16605 }
16606}
16607
16608// FastEmit functions for X86ISD::FXOR.
16609
16610Register fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
16611 if (RetVT.SimpleTy != MVT::f128)
16612 return Register();
16613 if ((Subtarget->hasVLX())) {
16614 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16615 }
16616 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16617 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16618 }
16619 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16620 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16621 }
16622 return Register();
16623}
16624
16625Register fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16626 if (RetVT.SimpleTy != MVT::v4f32)
16627 return Register();
16628 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16629}
16630
16631Register fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16632 switch (VT.SimpleTy) {
16633 case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op1);
16634 case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op1);
16635 default: return Register();
16636 }
16637}
16638
16639// FastEmit functions for X86ISD::GF2P8MULB.
16640
16641Register fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
16642 if (RetVT.SimpleTy != MVT::v16i8)
16643 return Register();
16644 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
16645 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16646 }
16647 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
16648 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
16649 }
16650 if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16651 return fastEmitInst_rr(MachineInstOpcode: X86::GF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
16652 }
16653 return Register();
16654}
16655
16656Register fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
16657 if (RetVT.SimpleTy != MVT::v32i8)
16658 return Register();
16659 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
16660 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16661 }
16662 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
16663 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBYrr, RC: &X86::VR256RegClass, Op0, Op1);
16664 }
16665 return Register();
16666}
16667
16668Register fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
16669 if (RetVT.SimpleTy != MVT::v64i8)
16670 return Register();
16671 if ((Subtarget->hasAVX512()) && (Subtarget->hasGFNI())) {
16672 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZrr, RC: &X86::VR512RegClass, Op0, Op1);
16673 }
16674 return Register();
16675}
16676
16677Register fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16678 switch (VT.SimpleTy) {
16679 case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op1);
16680 case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op1);
16681 case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op1);
16682 default: return Register();
16683 }
16684}
16685
16686// FastEmit functions for X86ISD::HADD.
16687
16688Register fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
16689 if (RetVT.SimpleTy != MVT::v8i16)
16690 return Register();
16691 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16692 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
16693 }
16694 if ((Subtarget->hasAVX())) {
16695 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
16696 }
16697 return Register();
16698}
16699
16700Register fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
16701 if (RetVT.SimpleTy != MVT::v16i16)
16702 return Register();
16703 if ((Subtarget->hasAVX2())) {
16704 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16705 }
16706 return Register();
16707}
16708
16709Register fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
16710 if (RetVT.SimpleTy != MVT::v4i32)
16711 return Register();
16712 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16713 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
16714 }
16715 if ((Subtarget->hasAVX())) {
16716 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
16717 }
16718 return Register();
16719}
16720
16721Register fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
16722 if (RetVT.SimpleTy != MVT::v8i32)
16723 return Register();
16724 if ((Subtarget->hasAVX2())) {
16725 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16726 }
16727 return Register();
16728}
16729
16730Register fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16731 switch (VT.SimpleTy) {
16732 case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op1);
16733 case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op1);
16734 case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op1);
16735 case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op1);
16736 default: return Register();
16737 }
16738}
16739
16740// FastEmit functions for X86ISD::HADDS.
16741
16742Register fastEmit_X86ISD_HADDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
16743 if (RetVT.SimpleTy != MVT::v8i16)
16744 return Register();
16745 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16746 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
16747 }
16748 if ((Subtarget->hasAVX())) {
16749 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
16750 }
16751 return Register();
16752}
16753
16754Register fastEmit_X86ISD_HADDS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
16755 if (RetVT.SimpleTy != MVT::v16i16)
16756 return Register();
16757 if ((Subtarget->hasAVX2())) {
16758 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16759 }
16760 return Register();
16761}
16762
16763Register fastEmit_X86ISD_HADDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16764 switch (VT.SimpleTy) {
16765 case MVT::v8i16: return fastEmit_X86ISD_HADDS_MVT_v8i16_rr(RetVT, Op0, Op1);
16766 case MVT::v16i16: return fastEmit_X86ISD_HADDS_MVT_v16i16_rr(RetVT, Op0, Op1);
16767 default: return Register();
16768 }
16769}
16770
16771// FastEmit functions for X86ISD::HSUB.
16772
16773Register fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
16774 if (RetVT.SimpleTy != MVT::v8i16)
16775 return Register();
16776 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16777 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
16778 }
16779 if ((Subtarget->hasAVX())) {
16780 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
16781 }
16782 return Register();
16783}
16784
16785Register fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
16786 if (RetVT.SimpleTy != MVT::v16i16)
16787 return Register();
16788 if ((Subtarget->hasAVX2())) {
16789 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16790 }
16791 return Register();
16792}
16793
16794Register fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
16795 if (RetVT.SimpleTy != MVT::v4i32)
16796 return Register();
16797 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16798 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
16799 }
16800 if ((Subtarget->hasAVX())) {
16801 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
16802 }
16803 return Register();
16804}
16805
16806Register fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
16807 if (RetVT.SimpleTy != MVT::v8i32)
16808 return Register();
16809 if ((Subtarget->hasAVX2())) {
16810 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16811 }
16812 return Register();
16813}
16814
16815Register fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16816 switch (VT.SimpleTy) {
16817 case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op1);
16818 case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op1);
16819 case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op1);
16820 case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op1);
16821 default: return Register();
16822 }
16823}
16824
16825// FastEmit functions for X86ISD::HSUBS.
16826
16827Register fastEmit_X86ISD_HSUBS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
16828 if (RetVT.SimpleTy != MVT::v8i16)
16829 return Register();
16830 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
16831 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
16832 }
16833 if ((Subtarget->hasAVX())) {
16834 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
16835 }
16836 return Register();
16837}
16838
16839Register fastEmit_X86ISD_HSUBS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
16840 if (RetVT.SimpleTy != MVT::v16i16)
16841 return Register();
16842 if ((Subtarget->hasAVX2())) {
16843 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
16844 }
16845 return Register();
16846}
16847
16848Register fastEmit_X86ISD_HSUBS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16849 switch (VT.SimpleTy) {
16850 case MVT::v8i16: return fastEmit_X86ISD_HSUBS_MVT_v8i16_rr(RetVT, Op0, Op1);
16851 case MVT::v16i16: return fastEmit_X86ISD_HSUBS_MVT_v16i16_rr(RetVT, Op0, Op1);
16852 default: return Register();
16853 }
16854}
16855
16856// FastEmit functions for X86ISD::KADD.
16857
16858Register fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
16859 if (RetVT.SimpleTy != MVT::v8i1)
16860 return Register();
16861 if ((Subtarget->hasDQI())) {
16862 return fastEmitInst_rr(MachineInstOpcode: X86::KADDBkk, RC: &X86::VK8RegClass, Op0, Op1);
16863 }
16864 return Register();
16865}
16866
16867Register fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
16868 if (RetVT.SimpleTy != MVT::v16i1)
16869 return Register();
16870 if ((Subtarget->hasDQI())) {
16871 return fastEmitInst_rr(MachineInstOpcode: X86::KADDWkk, RC: &X86::VK16RegClass, Op0, Op1);
16872 }
16873 return Register();
16874}
16875
16876Register fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
16877 if (RetVT.SimpleTy != MVT::v32i1)
16878 return Register();
16879 if ((Subtarget->hasBWI())) {
16880 return fastEmitInst_rr(MachineInstOpcode: X86::KADDDkk, RC: &X86::VK32RegClass, Op0, Op1);
16881 }
16882 return Register();
16883}
16884
16885Register fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
16886 if (RetVT.SimpleTy != MVT::v64i1)
16887 return Register();
16888 if ((Subtarget->hasBWI())) {
16889 return fastEmitInst_rr(MachineInstOpcode: X86::KADDQkk, RC: &X86::VK64RegClass, Op0, Op1);
16890 }
16891 return Register();
16892}
16893
16894Register fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16895 switch (VT.SimpleTy) {
16896 case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op1);
16897 case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op1);
16898 case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op1);
16899 case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op1);
16900 default: return Register();
16901 }
16902}
16903
16904// FastEmit functions for X86ISD::KORTEST.
16905
16906Register fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
16907 if (RetVT.SimpleTy != MVT::i32)
16908 return Register();
16909 if ((Subtarget->hasDQI())) {
16910 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTBkk, RC: &X86::VK8RegClass, Op0, Op1);
16911 }
16912 return Register();
16913}
16914
16915Register fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
16916 if (RetVT.SimpleTy != MVT::i32)
16917 return Register();
16918 if ((Subtarget->hasAVX512())) {
16919 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTWkk, RC: &X86::VK16RegClass, Op0, Op1);
16920 }
16921 return Register();
16922}
16923
16924Register fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
16925 if (RetVT.SimpleTy != MVT::i32)
16926 return Register();
16927 if ((Subtarget->hasBWI())) {
16928 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTDkk, RC: &X86::VK32RegClass, Op0, Op1);
16929 }
16930 return Register();
16931}
16932
16933Register fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
16934 if (RetVT.SimpleTy != MVT::i32)
16935 return Register();
16936 if ((Subtarget->hasBWI())) {
16937 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTQkk, RC: &X86::VK64RegClass, Op0, Op1);
16938 }
16939 return Register();
16940}
16941
16942Register fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16943 switch (VT.SimpleTy) {
16944 case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
16945 case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
16946 case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
16947 case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
16948 default: return Register();
16949 }
16950}
16951
16952// FastEmit functions for X86ISD::KTEST.
16953
16954Register fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
16955 if (RetVT.SimpleTy != MVT::i32)
16956 return Register();
16957 if ((Subtarget->hasDQI())) {
16958 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTBkk, RC: &X86::VK8RegClass, Op0, Op1);
16959 }
16960 return Register();
16961}
16962
16963Register fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
16964 if (RetVT.SimpleTy != MVT::i32)
16965 return Register();
16966 if ((Subtarget->hasDQI())) {
16967 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTWkk, RC: &X86::VK16RegClass, Op0, Op1);
16968 }
16969 return Register();
16970}
16971
16972Register fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
16973 if (RetVT.SimpleTy != MVT::i32)
16974 return Register();
16975 if ((Subtarget->hasBWI())) {
16976 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTDkk, RC: &X86::VK32RegClass, Op0, Op1);
16977 }
16978 return Register();
16979}
16980
16981Register fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
16982 if (RetVT.SimpleTy != MVT::i32)
16983 return Register();
16984 if ((Subtarget->hasBWI())) {
16985 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTQkk, RC: &X86::VK64RegClass, Op0, Op1);
16986 }
16987 return Register();
16988}
16989
16990Register fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16991 switch (VT.SimpleTy) {
16992 case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
16993 case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
16994 case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
16995 case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
16996 default: return Register();
16997 }
16998}
16999
17000// FastEmit functions for X86ISD::MOVHLPS.
17001
17002Register fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17003 if (RetVT.SimpleTy != MVT::v4f32)
17004 return Register();
17005 if ((Subtarget->hasAVX512())) {
17006 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17007 }
17008 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17009 return fastEmitInst_rr(MachineInstOpcode: X86::MOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17010 }
17011 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17012 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17013 }
17014 return Register();
17015}
17016
17017Register fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17018 switch (VT.SimpleTy) {
17019 case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op1);
17020 default: return Register();
17021 }
17022}
17023
17024// FastEmit functions for X86ISD::MOVLHPS.
17025
17026Register fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17027 if (RetVT.SimpleTy != MVT::v4f32)
17028 return Register();
17029 if ((Subtarget->hasAVX512())) {
17030 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17031 }
17032 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17033 return fastEmitInst_rr(MachineInstOpcode: X86::MOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17034 }
17035 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17036 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17037 }
17038 return Register();
17039}
17040
17041Register fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17042 switch (VT.SimpleTy) {
17043 case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op1);
17044 default: return Register();
17045 }
17046}
17047
17048// FastEmit functions for X86ISD::MOVSD.
17049
17050Register fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17051 if (RetVT.SimpleTy != MVT::v2f64)
17052 return Register();
17053 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
17054 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17055 }
17056 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17057 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
17058 }
17059 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17060 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
17061 }
17062 return Register();
17063}
17064
17065Register fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17066 switch (VT.SimpleTy) {
17067 case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op1);
17068 default: return Register();
17069 }
17070}
17071
17072// FastEmit functions for X86ISD::MOVSH.
17073
17074Register fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17075 if (RetVT.SimpleTy != MVT::v8f16)
17076 return Register();
17077 if ((Subtarget->hasFP16())) {
17078 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17079 }
17080 return Register();
17081}
17082
17083Register fastEmit_X86ISD_MOVSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17084 switch (VT.SimpleTy) {
17085 case MVT::v8f16: return fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(RetVT, Op0, Op1);
17086 default: return Register();
17087 }
17088}
17089
17090// FastEmit functions for X86ISD::MOVSS.
17091
17092Register fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17093 if (RetVT.SimpleTy != MVT::v4f32)
17094 return Register();
17095 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
17096 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17097 }
17098 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17099 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
17100 }
17101 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17102 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
17103 }
17104 return Register();
17105}
17106
17107Register fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17108 switch (VT.SimpleTy) {
17109 case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op1);
17110 default: return Register();
17111 }
17112}
17113
17114// FastEmit functions for X86ISD::MULHRS.
17115
17116Register fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17117 if (RetVT.SimpleTy != MVT::v8i16)
17118 return Register();
17119 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17120 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17121 }
17122 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17123 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17124 }
17125 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17126 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17127 }
17128 return Register();
17129}
17130
17131Register fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17132 if (RetVT.SimpleTy != MVT::v16i16)
17133 return Register();
17134 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17135 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17136 }
17137 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17138 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17139 }
17140 return Register();
17141}
17142
17143Register fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17144 if (RetVT.SimpleTy != MVT::v32i16)
17145 return Register();
17146 if ((Subtarget->hasBWI())) {
17147 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17148 }
17149 return Register();
17150}
17151
17152Register fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17153 switch (VT.SimpleTy) {
17154 case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op1);
17155 case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op1);
17156 case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op1);
17157 default: return Register();
17158 }
17159}
17160
17161// FastEmit functions for X86ISD::MULTISHIFT.
17162
17163Register fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17164 if (RetVT.SimpleTy != MVT::v16i8)
17165 return Register();
17166 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17167 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17168 }
17169 return Register();
17170}
17171
17172Register fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17173 if (RetVT.SimpleTy != MVT::v32i8)
17174 return Register();
17175 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17176 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17177 }
17178 return Register();
17179}
17180
17181Register fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17182 if (RetVT.SimpleTy != MVT::v64i8)
17183 return Register();
17184 if ((Subtarget->hasVBMI())) {
17185 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17186 }
17187 return Register();
17188}
17189
17190Register fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17191 switch (VT.SimpleTy) {
17192 case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op1);
17193 case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op1);
17194 case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op1);
17195 default: return Register();
17196 }
17197}
17198
17199// FastEmit functions for X86ISD::PACKSS.
17200
17201Register fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17202 if (RetVT.SimpleTy != MVT::v16i8)
17203 return Register();
17204 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17205 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17206 }
17207 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17208 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17209 }
17210 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17211 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17212 }
17213 return Register();
17214}
17215
17216Register fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17217 if (RetVT.SimpleTy != MVT::v32i8)
17218 return Register();
17219 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17220 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17221 }
17222 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17223 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17224 }
17225 return Register();
17226}
17227
17228Register fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17229 if (RetVT.SimpleTy != MVT::v64i8)
17230 return Register();
17231 if ((Subtarget->hasBWI())) {
17232 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17233 }
17234 return Register();
17235}
17236
17237Register fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17238 if (RetVT.SimpleTy != MVT::v8i16)
17239 return Register();
17240 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17241 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17242 }
17243 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17244 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17245 }
17246 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17247 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17248 }
17249 return Register();
17250}
17251
17252Register fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17253 if (RetVT.SimpleTy != MVT::v16i16)
17254 return Register();
17255 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17256 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17257 }
17258 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17259 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17260 }
17261 return Register();
17262}
17263
17264Register fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
17265 if (RetVT.SimpleTy != MVT::v32i16)
17266 return Register();
17267 if ((Subtarget->hasBWI())) {
17268 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17269 }
17270 return Register();
17271}
17272
17273Register fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17274 switch (VT.SimpleTy) {
17275 case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op1);
17276 case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op1);
17277 case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op1);
17278 case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op1);
17279 case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op1);
17280 case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op1);
17281 default: return Register();
17282 }
17283}
17284
17285// FastEmit functions for X86ISD::PACKUS.
17286
17287Register fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17288 if (RetVT.SimpleTy != MVT::v16i8)
17289 return Register();
17290 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17291 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17292 }
17293 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17294 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17295 }
17296 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17297 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17298 }
17299 return Register();
17300}
17301
17302Register fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17303 if (RetVT.SimpleTy != MVT::v32i8)
17304 return Register();
17305 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17306 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17307 }
17308 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17309 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17310 }
17311 return Register();
17312}
17313
17314Register fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17315 if (RetVT.SimpleTy != MVT::v64i8)
17316 return Register();
17317 if ((Subtarget->hasBWI())) {
17318 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17319 }
17320 return Register();
17321}
17322
17323Register fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17324 if (RetVT.SimpleTy != MVT::v8i16)
17325 return Register();
17326 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17327 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17328 }
17329 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17330 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17331 }
17332 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17333 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17334 }
17335 return Register();
17336}
17337
17338Register fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17339 if (RetVT.SimpleTy != MVT::v16i16)
17340 return Register();
17341 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17342 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17343 }
17344 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17345 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17346 }
17347 return Register();
17348}
17349
17350Register fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
17351 if (RetVT.SimpleTy != MVT::v32i16)
17352 return Register();
17353 if ((Subtarget->hasBWI())) {
17354 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17355 }
17356 return Register();
17357}
17358
17359Register fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17360 switch (VT.SimpleTy) {
17361 case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op1);
17362 case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op1);
17363 case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op1);
17364 case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op1);
17365 case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op1);
17366 case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op1);
17367 default: return Register();
17368 }
17369}
17370
17371// FastEmit functions for X86ISD::PCMPEQ.
17372
17373Register fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17374 if (RetVT.SimpleTy != MVT::v16i8)
17375 return Register();
17376 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17377 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
17378 }
17379 if ((Subtarget->hasAVX()) && (true)) {
17380 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
17381 }
17382 return Register();
17383}
17384
17385Register fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17386 if (RetVT.SimpleTy != MVT::v32i8)
17387 return Register();
17388 if ((Subtarget->hasAVX2()) && (true)) {
17389 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17390 }
17391 return Register();
17392}
17393
17394Register fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17395 if (RetVT.SimpleTy != MVT::v8i16)
17396 return Register();
17397 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17398 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
17399 }
17400 if ((Subtarget->hasAVX()) && (true)) {
17401 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
17402 }
17403 return Register();
17404}
17405
17406Register fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17407 if (RetVT.SimpleTy != MVT::v16i16)
17408 return Register();
17409 if ((Subtarget->hasAVX2()) && (true)) {
17410 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17411 }
17412 return Register();
17413}
17414
17415Register fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17416 if (RetVT.SimpleTy != MVT::v4i32)
17417 return Register();
17418 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17419 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
17420 }
17421 if ((Subtarget->hasAVX()) && (true)) {
17422 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
17423 }
17424 return Register();
17425}
17426
17427Register fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17428 if (RetVT.SimpleTy != MVT::v8i32)
17429 return Register();
17430 if ((Subtarget->hasAVX2()) && (true)) {
17431 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17432 }
17433 return Register();
17434}
17435
17436Register fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17437 if (RetVT.SimpleTy != MVT::v2i64)
17438 return Register();
17439 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17440 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
17441 }
17442 if ((Subtarget->hasAVX())) {
17443 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
17444 }
17445 return Register();
17446}
17447
17448Register fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17449 if (RetVT.SimpleTy != MVT::v4i64)
17450 return Register();
17451 if ((Subtarget->hasAVX2())) {
17452 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17453 }
17454 return Register();
17455}
17456
17457Register fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17458 switch (VT.SimpleTy) {
17459 case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op1);
17460 case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op1);
17461 case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op1);
17462 case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op1);
17463 case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op1);
17464 case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op1);
17465 case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17466 case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17467 default: return Register();
17468 }
17469}
17470
17471// FastEmit functions for X86ISD::PCMPGT.
17472
17473Register fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17474 if (RetVT.SimpleTy != MVT::v16i8)
17475 return Register();
17476 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17477 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
17478 }
17479 if ((Subtarget->hasAVX()) && (true)) {
17480 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
17481 }
17482 return Register();
17483}
17484
17485Register fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17486 if (RetVT.SimpleTy != MVT::v32i8)
17487 return Register();
17488 if ((Subtarget->hasAVX2()) && (true)) {
17489 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17490 }
17491 return Register();
17492}
17493
17494Register fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17495 if (RetVT.SimpleTy != MVT::v8i16)
17496 return Register();
17497 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17498 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
17499 }
17500 if ((Subtarget->hasAVX()) && (true)) {
17501 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
17502 }
17503 return Register();
17504}
17505
17506Register fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17507 if (RetVT.SimpleTy != MVT::v16i16)
17508 return Register();
17509 if ((Subtarget->hasAVX2()) && (true)) {
17510 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17511 }
17512 return Register();
17513}
17514
17515Register fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17516 if (RetVT.SimpleTy != MVT::v4i32)
17517 return Register();
17518 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17519 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
17520 }
17521 if ((Subtarget->hasAVX()) && (true)) {
17522 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
17523 }
17524 return Register();
17525}
17526
17527Register fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17528 if (RetVT.SimpleTy != MVT::v8i32)
17529 return Register();
17530 if ((Subtarget->hasAVX2()) && (true)) {
17531 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17532 }
17533 return Register();
17534}
17535
17536Register fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17537 if (RetVT.SimpleTy != MVT::v2i64)
17538 return Register();
17539 if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) {
17540 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
17541 }
17542 if ((Subtarget->hasAVX())) {
17543 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
17544 }
17545 return Register();
17546}
17547
17548Register fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17549 if (RetVT.SimpleTy != MVT::v4i64)
17550 return Register();
17551 if ((Subtarget->hasAVX2())) {
17552 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17553 }
17554 return Register();
17555}
17556
17557Register fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17558 switch (VT.SimpleTy) {
17559 case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op1);
17560 case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op1);
17561 case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op1);
17562 case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op1);
17563 case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op1);
17564 case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op1);
17565 case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op1);
17566 case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op1);
17567 default: return Register();
17568 }
17569}
17570
17571// FastEmit functions for X86ISD::PDEP.
17572
17573Register fastEmit_X86ISD_PDEP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
17574 if (RetVT.SimpleTy != MVT::i32)
17575 return Register();
17576 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
17577 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
17578 }
17579 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
17580 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr, RC: &X86::GR32RegClass, Op0, Op1);
17581 }
17582 return Register();
17583}
17584
17585Register fastEmit_X86ISD_PDEP_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
17586 if (RetVT.SimpleTy != MVT::i64)
17587 return Register();
17588 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
17589 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
17590 }
17591 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
17592 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr, RC: &X86::GR64RegClass, Op0, Op1);
17593 }
17594 return Register();
17595}
17596
17597Register fastEmit_X86ISD_PDEP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17598 switch (VT.SimpleTy) {
17599 case MVT::i32: return fastEmit_X86ISD_PDEP_MVT_i32_rr(RetVT, Op0, Op1);
17600 case MVT::i64: return fastEmit_X86ISD_PDEP_MVT_i64_rr(RetVT, Op0, Op1);
17601 default: return Register();
17602 }
17603}
17604
17605// FastEmit functions for X86ISD::PEXT.
17606
17607Register fastEmit_X86ISD_PEXT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
17608 if (RetVT.SimpleTy != MVT::i32)
17609 return Register();
17610 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
17611 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
17612 }
17613 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
17614 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr, RC: &X86::GR32RegClass, Op0, Op1);
17615 }
17616 return Register();
17617}
17618
17619Register fastEmit_X86ISD_PEXT_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
17620 if (RetVT.SimpleTy != MVT::i64)
17621 return Register();
17622 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
17623 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
17624 }
17625 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
17626 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr, RC: &X86::GR64RegClass, Op0, Op1);
17627 }
17628 return Register();
17629}
17630
17631Register fastEmit_X86ISD_PEXT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17632 switch (VT.SimpleTy) {
17633 case MVT::i32: return fastEmit_X86ISD_PEXT_MVT_i32_rr(RetVT, Op0, Op1);
17634 case MVT::i64: return fastEmit_X86ISD_PEXT_MVT_i64_rr(RetVT, Op0, Op1);
17635 default: return Register();
17636 }
17637}
17638
17639// FastEmit functions for X86ISD::PMULDQ.
17640
17641Register fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17642 if (RetVT.SimpleTy != MVT::v2i64)
17643 return Register();
17644 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17645 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17646 }
17647 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17648 return fastEmitInst_rr(MachineInstOpcode: X86::PMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17649 }
17650 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17651 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17652 }
17653 return Register();
17654}
17655
17656Register fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17657 if (RetVT.SimpleTy != MVT::v4i64)
17658 return Register();
17659 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17660 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17661 }
17662 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17663 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17664 }
17665 return Register();
17666}
17667
17668Register fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
17669 if (RetVT.SimpleTy != MVT::v8i64)
17670 return Register();
17671 if ((Subtarget->hasAVX512())) {
17672 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17673 }
17674 return Register();
17675}
17676
17677Register fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17678 switch (VT.SimpleTy) {
17679 case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17680 case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17681 case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
17682 default: return Register();
17683 }
17684}
17685
17686// FastEmit functions for X86ISD::PMULUDQ.
17687
17688Register fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17689 if (RetVT.SimpleTy != MVT::v2i64)
17690 return Register();
17691 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17692 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17693 }
17694 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17695 return fastEmitInst_rr(MachineInstOpcode: X86::PMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17696 }
17697 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17698 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
17699 }
17700 return Register();
17701}
17702
17703Register fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17704 if (RetVT.SimpleTy != MVT::v4i64)
17705 return Register();
17706 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17707 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17708 }
17709 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17710 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17711 }
17712 return Register();
17713}
17714
17715Register fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
17716 if (RetVT.SimpleTy != MVT::v8i64)
17717 return Register();
17718 if ((Subtarget->hasAVX512())) {
17719 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
17720 }
17721 return Register();
17722}
17723
17724Register fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17725 switch (VT.SimpleTy) {
17726 case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17727 case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17728 case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
17729 default: return Register();
17730 }
17731}
17732
17733// FastEmit functions for X86ISD::PSADBW.
17734
17735Register fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17736 if (RetVT.SimpleTy != MVT::v2i64)
17737 return Register();
17738 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17739 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17740 }
17741 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17742 return fastEmitInst_rr(MachineInstOpcode: X86::PSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17743 }
17744 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17745 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17746 }
17747 return Register();
17748}
17749
17750Register fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17751 if (RetVT.SimpleTy != MVT::v4i64)
17752 return Register();
17753 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17754 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17755 }
17756 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17757 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17758 }
17759 return Register();
17760}
17761
17762Register fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17763 if (RetVT.SimpleTy != MVT::v8i64)
17764 return Register();
17765 if ((Subtarget->hasBWI())) {
17766 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17767 }
17768 return Register();
17769}
17770
17771Register fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17772 switch (VT.SimpleTy) {
17773 case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op1);
17774 case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op1);
17775 case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op1);
17776 default: return Register();
17777 }
17778}
17779
17780// FastEmit functions for X86ISD::PSHUFB.
17781
17782Register fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17783 if (RetVT.SimpleTy != MVT::v16i8)
17784 return Register();
17785 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17786 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17787 }
17788 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17789 return fastEmitInst_rr(MachineInstOpcode: X86::PSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
17790 }
17791 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17792 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
17793 }
17794 return Register();
17795}
17796
17797Register fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17798 if (RetVT.SimpleTy != MVT::v32i8)
17799 return Register();
17800 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17801 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17802 }
17803 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17804 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17805 }
17806 return Register();
17807}
17808
17809Register fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17810 if (RetVT.SimpleTy != MVT::v64i8)
17811 return Register();
17812 if ((Subtarget->hasBWI())) {
17813 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17814 }
17815 return Register();
17816}
17817
17818Register fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17819 switch (VT.SimpleTy) {
17820 case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op1);
17821 case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op1);
17822 case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op1);
17823 default: return Register();
17824 }
17825}
17826
17827// FastEmit functions for X86ISD::PTEST.
17828
17829Register fastEmit_X86ISD_PTEST_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17830 if (RetVT.SimpleTy != MVT::i32)
17831 return Register();
17832 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17833 return fastEmitInst_rr(MachineInstOpcode: X86::PTESTrr, RC: &X86::VR128RegClass, Op0, Op1);
17834 }
17835 if ((Subtarget->hasAVX())) {
17836 return fastEmitInst_rr(MachineInstOpcode: X86::VPTESTrr, RC: &X86::VR128RegClass, Op0, Op1);
17837 }
17838 return Register();
17839}
17840
17841Register fastEmit_X86ISD_PTEST_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17842 if (RetVT.SimpleTy != MVT::i32)
17843 return Register();
17844 if ((Subtarget->hasAVX())) {
17845 return fastEmitInst_rr(MachineInstOpcode: X86::VPTESTYrr, RC: &X86::VR256RegClass, Op0, Op1);
17846 }
17847 return Register();
17848}
17849
17850Register fastEmit_X86ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17851 switch (VT.SimpleTy) {
17852 case MVT::v2i64: return fastEmit_X86ISD_PTEST_MVT_v2i64_rr(RetVT, Op0, Op1);
17853 case MVT::v4i64: return fastEmit_X86ISD_PTEST_MVT_v4i64_rr(RetVT, Op0, Op1);
17854 default: return Register();
17855 }
17856}
17857
17858// FastEmit functions for X86ISD::RCP14S.
17859
17860Register fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17861 if (RetVT.SimpleTy != MVT::v8f16)
17862 return Register();
17863 if ((Subtarget->hasFP16())) {
17864 return fastEmitInst_rr(MachineInstOpcode: X86::VRCPSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17865 }
17866 return Register();
17867}
17868
17869Register fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17870 if (RetVT.SimpleTy != MVT::v4f32)
17871 return Register();
17872 if ((Subtarget->hasAVX512())) {
17873 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17874 }
17875 return Register();
17876}
17877
17878Register fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17879 if (RetVT.SimpleTy != MVT::v2f64)
17880 return Register();
17881 if ((Subtarget->hasAVX512())) {
17882 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17883 }
17884 return Register();
17885}
17886
17887Register fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17888 switch (VT.SimpleTy) {
17889 case MVT::v8f16: return fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(RetVT, Op0, Op1);
17890 case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op1);
17891 case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op1);
17892 default: return Register();
17893 }
17894}
17895
17896// FastEmit functions for X86ISD::RSQRT14S.
17897
17898Register fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17899 if (RetVT.SimpleTy != MVT::v8f16)
17900 return Register();
17901 if ((Subtarget->hasFP16())) {
17902 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRTSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17903 }
17904 return Register();
17905}
17906
17907Register fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17908 if (RetVT.SimpleTy != MVT::v4f32)
17909 return Register();
17910 if ((Subtarget->hasAVX512())) {
17911 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17912 }
17913 return Register();
17914}
17915
17916Register fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17917 if (RetVT.SimpleTy != MVT::v2f64)
17918 return Register();
17919 if ((Subtarget->hasAVX512())) {
17920 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17921 }
17922 return Register();
17923}
17924
17925Register fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17926 switch (VT.SimpleTy) {
17927 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(RetVT, Op0, Op1);
17928 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op1);
17929 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op1);
17930 default: return Register();
17931 }
17932}
17933
17934// FastEmit functions for X86ISD::SCALEF.
17935
17936Register fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17937 if (RetVT.SimpleTy != MVT::v8f16)
17938 return Register();
17939 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17940 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17941 }
17942 return Register();
17943}
17944
17945Register fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
17946 if (RetVT.SimpleTy != MVT::v16f16)
17947 return Register();
17948 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17949 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17950 }
17951 return Register();
17952}
17953
17954Register fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
17955 if (RetVT.SimpleTy != MVT::v32f16)
17956 return Register();
17957 if ((Subtarget->hasFP16())) {
17958 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
17959 }
17960 return Register();
17961}
17962
17963Register fastEmit_X86ISD_SCALEF_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
17964 if (RetVT.SimpleTy != MVT::v8bf16)
17965 return Register();
17966 if ((Subtarget->hasAVX10_2())) {
17967 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17968 }
17969 return Register();
17970}
17971
17972Register fastEmit_X86ISD_SCALEF_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
17973 if (RetVT.SimpleTy != MVT::v16bf16)
17974 return Register();
17975 if ((Subtarget->hasAVX10_2())) {
17976 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17977 }
17978 return Register();
17979}
17980
17981Register fastEmit_X86ISD_SCALEF_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
17982 if (RetVT.SimpleTy != MVT::v32bf16)
17983 return Register();
17984 if ((Subtarget->hasAVX10_2())) {
17985 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
17986 }
17987 return Register();
17988}
17989
17990Register fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17991 if (RetVT.SimpleTy != MVT::v4f32)
17992 return Register();
17993 if ((Subtarget->hasVLX())) {
17994 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17995 }
17996 return Register();
17997}
17998
17999Register fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18000 if (RetVT.SimpleTy != MVT::v8f32)
18001 return Register();
18002 if ((Subtarget->hasVLX())) {
18003 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18004 }
18005 return Register();
18006}
18007
18008Register fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18009 if (RetVT.SimpleTy != MVT::v16f32)
18010 return Register();
18011 if ((Subtarget->hasAVX512())) {
18012 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18013 }
18014 return Register();
18015}
18016
18017Register fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18018 if (RetVT.SimpleTy != MVT::v2f64)
18019 return Register();
18020 if ((Subtarget->hasVLX())) {
18021 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18022 }
18023 return Register();
18024}
18025
18026Register fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18027 if (RetVT.SimpleTy != MVT::v4f64)
18028 return Register();
18029 if ((Subtarget->hasVLX())) {
18030 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18031 }
18032 return Register();
18033}
18034
18035Register fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18036 if (RetVT.SimpleTy != MVT::v8f64)
18037 return Register();
18038 if ((Subtarget->hasAVX512())) {
18039 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18040 }
18041 return Register();
18042}
18043
18044Register fastEmit_X86ISD_SCALEF_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18045 switch (VT.SimpleTy) {
18046 case MVT::v8f16: return fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(RetVT, Op0, Op1);
18047 case MVT::v16f16: return fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(RetVT, Op0, Op1);
18048 case MVT::v32f16: return fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(RetVT, Op0, Op1);
18049 case MVT::v8bf16: return fastEmit_X86ISD_SCALEF_MVT_v8bf16_rr(RetVT, Op0, Op1);
18050 case MVT::v16bf16: return fastEmit_X86ISD_SCALEF_MVT_v16bf16_rr(RetVT, Op0, Op1);
18051 case MVT::v32bf16: return fastEmit_X86ISD_SCALEF_MVT_v32bf16_rr(RetVT, Op0, Op1);
18052 case MVT::v4f32: return fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(RetVT, Op0, Op1);
18053 case MVT::v8f32: return fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(RetVT, Op0, Op1);
18054 case MVT::v16f32: return fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(RetVT, Op0, Op1);
18055 case MVT::v2f64: return fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(RetVT, Op0, Op1);
18056 case MVT::v4f64: return fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(RetVT, Op0, Op1);
18057 case MVT::v8f64: return fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(RetVT, Op0, Op1);
18058 default: return Register();
18059 }
18060}
18061
18062// FastEmit functions for X86ISD::SCALEFS.
18063
18064Register fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18065 if (RetVT.SimpleTy != MVT::v8f16)
18066 return Register();
18067 if ((Subtarget->hasFP16())) {
18068 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18069 }
18070 return Register();
18071}
18072
18073Register fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18074 if (RetVT.SimpleTy != MVT::v4f32)
18075 return Register();
18076 if ((Subtarget->hasAVX512())) {
18077 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18078 }
18079 return Register();
18080}
18081
18082Register fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18083 if (RetVT.SimpleTy != MVT::v2f64)
18084 return Register();
18085 if ((Subtarget->hasAVX512())) {
18086 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18087 }
18088 return Register();
18089}
18090
18091Register fastEmit_X86ISD_SCALEFS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18092 switch (VT.SimpleTy) {
18093 case MVT::v8f16: return fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(RetVT, Op0, Op1);
18094 case MVT::v4f32: return fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(RetVT, Op0, Op1);
18095 case MVT::v2f64: return fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(RetVT, Op0, Op1);
18096 default: return Register();
18097 }
18098}
18099
18100// FastEmit functions for X86ISD::STRICT_FCMP.
18101
18102Register fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18103 if (RetVT.SimpleTy != MVT::i32)
18104 return Register();
18105 if ((Subtarget->hasFP16())) {
18106 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18107 }
18108 return Register();
18109}
18110
18111Register fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18112 if (RetVT.SimpleTy != MVT::i32)
18113 return Register();
18114 if ((Subtarget->hasAVX512())) {
18115 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18116 }
18117 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18118 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18119 }
18120 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18121 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18122 }
18123 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
18124 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
18125 }
18126 return Register();
18127}
18128
18129Register fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18130 if (RetVT.SimpleTy != MVT::i32)
18131 return Register();
18132 if ((Subtarget->hasAVX512())) {
18133 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18134 }
18135 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18136 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18137 }
18138 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18139 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18140 }
18141 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
18142 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
18143 }
18144 return Register();
18145}
18146
18147Register fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18148 if (RetVT.SimpleTy != MVT::i32)
18149 return Register();
18150 if ((Subtarget->canUseCMOV())) {
18151 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
18152 }
18153 return Register();
18154}
18155
18156Register fastEmit_X86ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18157 switch (VT.SimpleTy) {
18158 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
18159 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
18160 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
18161 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
18162 default: return Register();
18163 }
18164}
18165
18166// FastEmit functions for X86ISD::STRICT_FCMPS.
18167
18168Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18169 if (RetVT.SimpleTy != MVT::i32)
18170 return Register();
18171 if ((Subtarget->hasFP16())) {
18172 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18173 }
18174 return Register();
18175}
18176
18177Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18178 if (RetVT.SimpleTy != MVT::i32)
18179 return Register();
18180 if ((Subtarget->hasAVX512())) {
18181 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18182 }
18183 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18184 return fastEmitInst_rr(MachineInstOpcode: X86::COMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18185 }
18186 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18187 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18188 }
18189 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
18190 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
18191 }
18192 return Register();
18193}
18194
18195Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18196 if (RetVT.SimpleTy != MVT::i32)
18197 return Register();
18198 if ((Subtarget->hasAVX512())) {
18199 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18200 }
18201 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18202 return fastEmitInst_rr(MachineInstOpcode: X86::COMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18203 }
18204 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18205 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18206 }
18207 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
18208 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
18209 }
18210 return Register();
18211}
18212
18213Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18214 if (RetVT.SimpleTy != MVT::i32)
18215 return Register();
18216 if ((Subtarget->canUseCMOV())) {
18217 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
18218 }
18219 return Register();
18220}
18221
18222Register fastEmit_X86ISD_STRICT_FCMPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18223 switch (VT.SimpleTy) {
18224 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(RetVT, Op0, Op1);
18225 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(RetVT, Op0, Op1);
18226 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(RetVT, Op0, Op1);
18227 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(RetVT, Op0, Op1);
18228 default: return Register();
18229 }
18230}
18231
18232// FastEmit functions for X86ISD::STRICT_FMAX.
18233
18234Register fastEmit_X86ISD_STRICT_FMAX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18235 if (RetVT.SimpleTy != MVT::f16)
18236 return Register();
18237 if ((Subtarget->hasFP16())) {
18238 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18239 }
18240 return Register();
18241}
18242
18243Register fastEmit_X86ISD_STRICT_FMAX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18244 if (RetVT.SimpleTy != MVT::f32)
18245 return Register();
18246 if ((Subtarget->hasAVX512())) {
18247 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18248 }
18249 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18250 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18251 }
18252 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18253 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18254 }
18255 return Register();
18256}
18257
18258Register fastEmit_X86ISD_STRICT_FMAX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18259 if (RetVT.SimpleTy != MVT::f64)
18260 return Register();
18261 if ((Subtarget->hasAVX512())) {
18262 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18263 }
18264 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18265 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18266 }
18267 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18268 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18269 }
18270 return Register();
18271}
18272
18273Register fastEmit_X86ISD_STRICT_FMAX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18274 if (RetVT.SimpleTy != MVT::v8f16)
18275 return Register();
18276 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18277 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18278 }
18279 return Register();
18280}
18281
18282Register fastEmit_X86ISD_STRICT_FMAX_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18283 if (RetVT.SimpleTy != MVT::v16f16)
18284 return Register();
18285 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18286 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18287 }
18288 return Register();
18289}
18290
18291Register fastEmit_X86ISD_STRICT_FMAX_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18292 if (RetVT.SimpleTy != MVT::v32f16)
18293 return Register();
18294 if ((Subtarget->hasFP16())) {
18295 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
18296 }
18297 return Register();
18298}
18299
18300Register fastEmit_X86ISD_STRICT_FMAX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18301 if (RetVT.SimpleTy != MVT::v4f32)
18302 return Register();
18303 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18304 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18305 }
18306 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18307 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18308 }
18309 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18310 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18311 }
18312 return Register();
18313}
18314
18315Register fastEmit_X86ISD_STRICT_FMAX_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18316 if (RetVT.SimpleTy != MVT::v8f32)
18317 return Register();
18318 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18319 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18320 }
18321 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18322 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18323 }
18324 return Register();
18325}
18326
18327Register fastEmit_X86ISD_STRICT_FMAX_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18328 if (RetVT.SimpleTy != MVT::v16f32)
18329 return Register();
18330 if ((Subtarget->hasAVX512())) {
18331 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18332 }
18333 return Register();
18334}
18335
18336Register fastEmit_X86ISD_STRICT_FMAX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18337 if (RetVT.SimpleTy != MVT::v2f64)
18338 return Register();
18339 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18340 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18341 }
18342 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18343 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18344 }
18345 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18346 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18347 }
18348 return Register();
18349}
18350
18351Register fastEmit_X86ISD_STRICT_FMAX_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18352 if (RetVT.SimpleTy != MVT::v4f64)
18353 return Register();
18354 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18355 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18356 }
18357 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18358 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18359 }
18360 return Register();
18361}
18362
18363Register fastEmit_X86ISD_STRICT_FMAX_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18364 if (RetVT.SimpleTy != MVT::v8f64)
18365 return Register();
18366 if ((Subtarget->hasAVX512())) {
18367 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18368 }
18369 return Register();
18370}
18371
18372Register fastEmit_X86ISD_STRICT_FMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18373 switch (VT.SimpleTy) {
18374 case MVT::f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
18375 case MVT::f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
18376 case MVT::f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
18377 case MVT::v8f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
18378 case MVT::v16f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
18379 case MVT::v32f16: return fastEmit_X86ISD_STRICT_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
18380 case MVT::v4f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
18381 case MVT::v8f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
18382 case MVT::v16f32: return fastEmit_X86ISD_STRICT_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
18383 case MVT::v2f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
18384 case MVT::v4f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
18385 case MVT::v8f64: return fastEmit_X86ISD_STRICT_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
18386 default: return Register();
18387 }
18388}
18389
18390// FastEmit functions for X86ISD::STRICT_FMIN.
18391
18392Register fastEmit_X86ISD_STRICT_FMIN_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18393 if (RetVT.SimpleTy != MVT::f16)
18394 return Register();
18395 if ((Subtarget->hasFP16())) {
18396 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18397 }
18398 return Register();
18399}
18400
18401Register fastEmit_X86ISD_STRICT_FMIN_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18402 if (RetVT.SimpleTy != MVT::f32)
18403 return Register();
18404 if ((Subtarget->hasAVX512())) {
18405 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18406 }
18407 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18408 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18409 }
18410 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18411 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
18412 }
18413 return Register();
18414}
18415
18416Register fastEmit_X86ISD_STRICT_FMIN_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18417 if (RetVT.SimpleTy != MVT::f64)
18418 return Register();
18419 if ((Subtarget->hasAVX512())) {
18420 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18421 }
18422 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18423 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18424 }
18425 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18426 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
18427 }
18428 return Register();
18429}
18430
18431Register fastEmit_X86ISD_STRICT_FMIN_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18432 if (RetVT.SimpleTy != MVT::v8f16)
18433 return Register();
18434 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18435 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18436 }
18437 return Register();
18438}
18439
18440Register fastEmit_X86ISD_STRICT_FMIN_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18441 if (RetVT.SimpleTy != MVT::v16f16)
18442 return Register();
18443 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18444 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18445 }
18446 return Register();
18447}
18448
18449Register fastEmit_X86ISD_STRICT_FMIN_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18450 if (RetVT.SimpleTy != MVT::v32f16)
18451 return Register();
18452 if ((Subtarget->hasFP16())) {
18453 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
18454 }
18455 return Register();
18456}
18457
18458Register fastEmit_X86ISD_STRICT_FMIN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18459 if (RetVT.SimpleTy != MVT::v4f32)
18460 return Register();
18461 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18462 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18463 }
18464 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18465 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18466 }
18467 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18468 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18469 }
18470 return Register();
18471}
18472
18473Register fastEmit_X86ISD_STRICT_FMIN_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18474 if (RetVT.SimpleTy != MVT::v8f32)
18475 return Register();
18476 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18477 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18478 }
18479 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18480 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18481 }
18482 return Register();
18483}
18484
18485Register fastEmit_X86ISD_STRICT_FMIN_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18486 if (RetVT.SimpleTy != MVT::v16f32)
18487 return Register();
18488 if ((Subtarget->hasAVX512())) {
18489 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18490 }
18491 return Register();
18492}
18493
18494Register fastEmit_X86ISD_STRICT_FMIN_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18495 if (RetVT.SimpleTy != MVT::v2f64)
18496 return Register();
18497 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18498 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18499 }
18500 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18501 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18502 }
18503 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18504 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18505 }
18506 return Register();
18507}
18508
18509Register fastEmit_X86ISD_STRICT_FMIN_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18510 if (RetVT.SimpleTy != MVT::v4f64)
18511 return Register();
18512 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18513 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18514 }
18515 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18516 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18517 }
18518 return Register();
18519}
18520
18521Register fastEmit_X86ISD_STRICT_FMIN_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18522 if (RetVT.SimpleTy != MVT::v8f64)
18523 return Register();
18524 if ((Subtarget->hasAVX512())) {
18525 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18526 }
18527 return Register();
18528}
18529
18530Register fastEmit_X86ISD_STRICT_FMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18531 switch (VT.SimpleTy) {
18532 case MVT::f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
18533 case MVT::f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
18534 case MVT::f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
18535 case MVT::v8f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
18536 case MVT::v16f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
18537 case MVT::v32f16: return fastEmit_X86ISD_STRICT_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
18538 case MVT::v4f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
18539 case MVT::v8f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
18540 case MVT::v16f32: return fastEmit_X86ISD_STRICT_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
18541 case MVT::v2f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
18542 case MVT::v4f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
18543 case MVT::v8f64: return fastEmit_X86ISD_STRICT_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
18544 default: return Register();
18545 }
18546}
18547
18548// FastEmit functions for X86ISD::STRICT_FP80_ADD.
18549
18550Register fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18551 if (RetVT.SimpleTy != MVT::f80)
18552 return Register();
18553 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
18554}
18555
18556Register fastEmit_X86ISD_STRICT_FP80_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18557 switch (VT.SimpleTy) {
18558 case MVT::f80: return fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
18559 default: return Register();
18560 }
18561}
18562
18563// FastEmit functions for X86ISD::TESTP.
18564
18565Register fastEmit_X86ISD_TESTP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18566 if (RetVT.SimpleTy != MVT::i32)
18567 return Register();
18568 if ((Subtarget->hasAVX())) {
18569 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18570 }
18571 return Register();
18572}
18573
18574Register fastEmit_X86ISD_TESTP_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18575 if (RetVT.SimpleTy != MVT::i32)
18576 return Register();
18577 if ((Subtarget->hasAVX())) {
18578 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18579 }
18580 return Register();
18581}
18582
18583Register fastEmit_X86ISD_TESTP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18584 if (RetVT.SimpleTy != MVT::i32)
18585 return Register();
18586 if ((Subtarget->hasAVX())) {
18587 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18588 }
18589 return Register();
18590}
18591
18592Register fastEmit_X86ISD_TESTP_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18593 if (RetVT.SimpleTy != MVT::i32)
18594 return Register();
18595 if ((Subtarget->hasAVX())) {
18596 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18597 }
18598 return Register();
18599}
18600
18601Register fastEmit_X86ISD_TESTP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18602 switch (VT.SimpleTy) {
18603 case MVT::v4f32: return fastEmit_X86ISD_TESTP_MVT_v4f32_rr(RetVT, Op0, Op1);
18604 case MVT::v8f32: return fastEmit_X86ISD_TESTP_MVT_v8f32_rr(RetVT, Op0, Op1);
18605 case MVT::v2f64: return fastEmit_X86ISD_TESTP_MVT_v2f64_rr(RetVT, Op0, Op1);
18606 case MVT::v4f64: return fastEmit_X86ISD_TESTP_MVT_v4f64_rr(RetVT, Op0, Op1);
18607 default: return Register();
18608 }
18609}
18610
18611// FastEmit functions for X86ISD::UCOMI.
18612
18613Register fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18614 if (RetVT.SimpleTy != MVT::i32)
18615 return Register();
18616 if ((Subtarget->hasFP16())) {
18617 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18618 }
18619 return Register();
18620}
18621
18622Register fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18623 if (RetVT.SimpleTy != MVT::i32)
18624 return Register();
18625 if ((Subtarget->hasAVX512())) {
18626 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18627 }
18628 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18629 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18630 }
18631 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18632 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18633 }
18634 return Register();
18635}
18636
18637Register fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18638 if (RetVT.SimpleTy != MVT::i32)
18639 return Register();
18640 if ((Subtarget->hasAVX512())) {
18641 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18642 }
18643 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18644 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18645 }
18646 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18647 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18648 }
18649 return Register();
18650}
18651
18652Register fastEmit_X86ISD_UCOMI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18653 switch (VT.SimpleTy) {
18654 case MVT::v8f16: return fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(RetVT, Op0, Op1);
18655 case MVT::v4f32: return fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(RetVT, Op0, Op1);
18656 case MVT::v2f64: return fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(RetVT, Op0, Op1);
18657 default: return Register();
18658 }
18659}
18660
18661// FastEmit functions for X86ISD::UCOMX.
18662
18663Register fastEmit_X86ISD_UCOMX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18664 if (RetVT.SimpleTy != MVT::i32)
18665 return Register();
18666 if ((Subtarget->hasAVX10_2())) {
18667 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18668 }
18669 return Register();
18670}
18671
18672Register fastEmit_X86ISD_UCOMX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18673 if (RetVT.SimpleTy != MVT::i32)
18674 return Register();
18675 if ((Subtarget->hasAVX10_2())) {
18676 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18677 }
18678 return Register();
18679}
18680
18681Register fastEmit_X86ISD_UCOMX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18682 if (RetVT.SimpleTy != MVT::i32)
18683 return Register();
18684 if ((Subtarget->hasAVX10_2())) {
18685 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18686 }
18687 return Register();
18688}
18689
18690Register fastEmit_X86ISD_UCOMX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18691 if (RetVT.SimpleTy != MVT::i32)
18692 return Register();
18693 if ((Subtarget->hasAVX10_2())) {
18694 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18695 }
18696 return Register();
18697}
18698
18699Register fastEmit_X86ISD_UCOMX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18700 if (RetVT.SimpleTy != MVT::i32)
18701 return Register();
18702 if ((Subtarget->hasAVX10_2())) {
18703 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18704 }
18705 return Register();
18706}
18707
18708Register fastEmit_X86ISD_UCOMX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18709 if (RetVT.SimpleTy != MVT::i32)
18710 return Register();
18711 if ((Subtarget->hasAVX10_2())) {
18712 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18713 }
18714 return Register();
18715}
18716
18717Register fastEmit_X86ISD_UCOMX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18718 switch (VT.SimpleTy) {
18719 case MVT::f16: return fastEmit_X86ISD_UCOMX_MVT_f16_rr(RetVT, Op0, Op1);
18720 case MVT::f32: return fastEmit_X86ISD_UCOMX_MVT_f32_rr(RetVT, Op0, Op1);
18721 case MVT::f64: return fastEmit_X86ISD_UCOMX_MVT_f64_rr(RetVT, Op0, Op1);
18722 case MVT::v8f16: return fastEmit_X86ISD_UCOMX_MVT_v8f16_rr(RetVT, Op0, Op1);
18723 case MVT::v4f32: return fastEmit_X86ISD_UCOMX_MVT_v4f32_rr(RetVT, Op0, Op1);
18724 case MVT::v2f64: return fastEmit_X86ISD_UCOMX_MVT_v2f64_rr(RetVT, Op0, Op1);
18725 default: return Register();
18726 }
18727}
18728
18729// FastEmit functions for X86ISD::UNPCKH.
18730
18731Register fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18732 if (RetVT.SimpleTy != MVT::v16i8)
18733 return Register();
18734 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18735 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18736 }
18737 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18738 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18739 }
18740 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18741 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18742 }
18743 return Register();
18744}
18745
18746Register fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18747 if (RetVT.SimpleTy != MVT::v32i8)
18748 return Register();
18749 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18750 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18751 }
18752 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18753 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
18754 }
18755 return Register();
18756}
18757
18758Register fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
18759 if (RetVT.SimpleTy != MVT::v64i8)
18760 return Register();
18761 if ((Subtarget->hasBWI())) {
18762 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
18763 }
18764 return Register();
18765}
18766
18767Register fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
18768 if (RetVT.SimpleTy != MVT::v8i16)
18769 return Register();
18770 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18771 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18772 }
18773 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18774 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18775 }
18776 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18777 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18778 }
18779 return Register();
18780}
18781
18782Register fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
18783 if (RetVT.SimpleTy != MVT::v16i16)
18784 return Register();
18785 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18786 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18787 }
18788 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18789 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18790 }
18791 return Register();
18792}
18793
18794Register fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
18795 if (RetVT.SimpleTy != MVT::v32i16)
18796 return Register();
18797 if ((Subtarget->hasBWI())) {
18798 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18799 }
18800 return Register();
18801}
18802
18803Register fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
18804 if (RetVT.SimpleTy != MVT::v4i32)
18805 return Register();
18806 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18807 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18808 }
18809 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18810 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18811 }
18812 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18813 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18814 }
18815 return Register();
18816}
18817
18818Register fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
18819 if (RetVT.SimpleTy != MVT::v8i32)
18820 return Register();
18821 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18822 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18823 }
18824 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18825 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18826 }
18827 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18828 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18829 }
18830 return Register();
18831}
18832
18833Register fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
18834 if (RetVT.SimpleTy != MVT::v16i32)
18835 return Register();
18836 if ((Subtarget->hasAVX512())) {
18837 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18838 }
18839 return Register();
18840}
18841
18842Register fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
18843 if (RetVT.SimpleTy != MVT::v2i64)
18844 return Register();
18845 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18846 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18847 }
18848 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18849 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18850 }
18851 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18852 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18853 }
18854 return Register();
18855}
18856
18857Register fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
18858 if (RetVT.SimpleTy != MVT::v4i64)
18859 return Register();
18860 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18861 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18862 }
18863 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18864 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18865 }
18866 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18867 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18868 }
18869 return Register();
18870}
18871
18872Register fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
18873 if (RetVT.SimpleTy != MVT::v8i64)
18874 return Register();
18875 if ((Subtarget->hasAVX512())) {
18876 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18877 }
18878 return Register();
18879}
18880
18881Register fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18882 if (RetVT.SimpleTy != MVT::v4f32)
18883 return Register();
18884 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18885 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18886 }
18887 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18888 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18889 }
18890 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18891 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18892 }
18893 return Register();
18894}
18895
18896Register fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18897 if (RetVT.SimpleTy != MVT::v8f32)
18898 return Register();
18899 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18900 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18901 }
18902 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18903 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18904 }
18905 return Register();
18906}
18907
18908Register fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18909 if (RetVT.SimpleTy != MVT::v16f32)
18910 return Register();
18911 if ((Subtarget->hasAVX512())) {
18912 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18913 }
18914 return Register();
18915}
18916
18917Register fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18918 if (RetVT.SimpleTy != MVT::v2f64)
18919 return Register();
18920 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18921 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18922 }
18923 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18924 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18925 }
18926 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18927 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18928 }
18929 return Register();
18930}
18931
18932Register fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18933 if (RetVT.SimpleTy != MVT::v4f64)
18934 return Register();
18935 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18936 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18937 }
18938 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18939 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18940 }
18941 return Register();
18942}
18943
18944Register fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18945 if (RetVT.SimpleTy != MVT::v8f64)
18946 return Register();
18947 if ((Subtarget->hasAVX512())) {
18948 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18949 }
18950 return Register();
18951}
18952
18953Register fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18954 switch (VT.SimpleTy) {
18955 case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op1);
18956 case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op1);
18957 case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op1);
18958 case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op1);
18959 case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op1);
18960 case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op1);
18961 case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op1);
18962 case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op1);
18963 case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op1);
18964 case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op1);
18965 case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op1);
18966 case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op1);
18967 case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op1);
18968 case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op1);
18969 case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op1);
18970 case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op1);
18971 case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op1);
18972 case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op1);
18973 default: return Register();
18974 }
18975}
18976
18977// FastEmit functions for X86ISD::UNPCKL.
18978
18979Register fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18980 if (RetVT.SimpleTy != MVT::v16i8)
18981 return Register();
18982 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18983 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18984 }
18985 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18986 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18987 }
18988 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18989 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18990 }
18991 return Register();
18992}
18993
18994Register fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18995 if (RetVT.SimpleTy != MVT::v32i8)
18996 return Register();
18997 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18998 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18999 }
19000 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19001 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
19002 }
19003 return Register();
19004}
19005
19006Register fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19007 if (RetVT.SimpleTy != MVT::v64i8)
19008 return Register();
19009 if ((Subtarget->hasBWI())) {
19010 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19011 }
19012 return Register();
19013}
19014
19015Register fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19016 if (RetVT.SimpleTy != MVT::v8i16)
19017 return Register();
19018 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19019 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19020 }
19021 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19022 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19023 }
19024 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19025 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19026 }
19027 return Register();
19028}
19029
19030Register fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19031 if (RetVT.SimpleTy != MVT::v16i16)
19032 return Register();
19033 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19034 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19035 }
19036 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19037 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19038 }
19039 return Register();
19040}
19041
19042Register fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19043 if (RetVT.SimpleTy != MVT::v32i16)
19044 return Register();
19045 if ((Subtarget->hasBWI())) {
19046 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19047 }
19048 return Register();
19049}
19050
19051Register fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19052 if (RetVT.SimpleTy != MVT::v4i32)
19053 return Register();
19054 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19055 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19056 }
19057 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19058 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19059 }
19060 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19061 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19062 }
19063 return Register();
19064}
19065
19066Register fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19067 if (RetVT.SimpleTy != MVT::v8i32)
19068 return Register();
19069 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
19070 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
19071 }
19072 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19073 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19074 }
19075 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19076 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
19077 }
19078 return Register();
19079}
19080
19081Register fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19082 if (RetVT.SimpleTy != MVT::v16i32)
19083 return Register();
19084 if ((Subtarget->hasAVX512())) {
19085 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19086 }
19087 return Register();
19088}
19089
19090Register fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19091 if (RetVT.SimpleTy != MVT::v2i64)
19092 return Register();
19093 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19094 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19095 }
19096 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19097 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19098 }
19099 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19100 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19101 }
19102 return Register();
19103}
19104
19105Register fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19106 if (RetVT.SimpleTy != MVT::v4i64)
19107 return Register();
19108 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
19109 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19110 }
19111 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19112 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19113 }
19114 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19115 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
19116 }
19117 return Register();
19118}
19119
19120Register fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19121 if (RetVT.SimpleTy != MVT::v8i64)
19122 return Register();
19123 if ((Subtarget->hasAVX512())) {
19124 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19125 }
19126 return Register();
19127}
19128
19129Register fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19130 if (RetVT.SimpleTy != MVT::v4f32)
19131 return Register();
19132 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19133 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19134 }
19135 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
19136 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
19137 }
19138 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19139 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
19140 }
19141 return Register();
19142}
19143
19144Register fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19145 if (RetVT.SimpleTy != MVT::v8f32)
19146 return Register();
19147 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19148 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19149 }
19150 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19151 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
19152 }
19153 return Register();
19154}
19155
19156Register fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19157 if (RetVT.SimpleTy != MVT::v16f32)
19158 return Register();
19159 if ((Subtarget->hasAVX512())) {
19160 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
19161 }
19162 return Register();
19163}
19164
19165Register fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
19166 if (RetVT.SimpleTy != MVT::v2f64)
19167 return Register();
19168 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19169 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19170 }
19171 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19172 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
19173 }
19174 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19175 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
19176 }
19177 return Register();
19178}
19179
19180Register fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
19181 if (RetVT.SimpleTy != MVT::v4f64)
19182 return Register();
19183 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19184 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19185 }
19186 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19187 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19188 }
19189 return Register();
19190}
19191
19192Register fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
19193 if (RetVT.SimpleTy != MVT::v8f64)
19194 return Register();
19195 if ((Subtarget->hasAVX512())) {
19196 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19197 }
19198 return Register();
19199}
19200
19201Register fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19202 switch (VT.SimpleTy) {
19203 case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op1);
19204 case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op1);
19205 case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op1);
19206 case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op1);
19207 case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op1);
19208 case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op1);
19209 case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op1);
19210 case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op1);
19211 case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op1);
19212 case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op1);
19213 case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op1);
19214 case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op1);
19215 case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op1);
19216 case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op1);
19217 case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op1);
19218 case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op1);
19219 case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op1);
19220 case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op1);
19221 default: return Register();
19222 }
19223}
19224
19225// FastEmit functions for X86ISD::VCVT2PH2BF8.
19226
19227Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19228 if (RetVT.SimpleTy != MVT::v16i8)
19229 return Register();
19230 if ((Subtarget->hasAVX10_2())) {
19231 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19232 }
19233 return Register();
19234}
19235
19236Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19237 if (RetVT.SimpleTy != MVT::v32i8)
19238 return Register();
19239 if ((Subtarget->hasAVX10_2())) {
19240 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19241 }
19242 return Register();
19243}
19244
19245Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19246 if (RetVT.SimpleTy != MVT::v64i8)
19247 return Register();
19248 if ((Subtarget->hasAVX10_2())) {
19249 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19250 }
19251 return Register();
19252}
19253
19254Register fastEmit_X86ISD_VCVT2PH2BF8_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19255 switch (VT.SimpleTy) {
19256 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v8f16_rr(RetVT, Op0, Op1);
19257 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v16f16_rr(RetVT, Op0, Op1);
19258 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v32f16_rr(RetVT, Op0, Op1);
19259 default: return Register();
19260 }
19261}
19262
19263// FastEmit functions for X86ISD::VCVT2PH2BF8S.
19264
19265Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19266 if (RetVT.SimpleTy != MVT::v16i8)
19267 return Register();
19268 if ((Subtarget->hasAVX10_2())) {
19269 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19270 }
19271 return Register();
19272}
19273
19274Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19275 if (RetVT.SimpleTy != MVT::v32i8)
19276 return Register();
19277 if ((Subtarget->hasAVX10_2())) {
19278 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19279 }
19280 return Register();
19281}
19282
19283Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19284 if (RetVT.SimpleTy != MVT::v64i8)
19285 return Register();
19286 if ((Subtarget->hasAVX10_2())) {
19287 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZrr, RC: &X86::VR512RegClass, Op0, Op1);
19288 }
19289 return Register();
19290}
19291
19292Register fastEmit_X86ISD_VCVT2PH2BF8S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19293 switch (VT.SimpleTy) {
19294 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v8f16_rr(RetVT, Op0, Op1);
19295 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v16f16_rr(RetVT, Op0, Op1);
19296 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v32f16_rr(RetVT, Op0, Op1);
19297 default: return Register();
19298 }
19299}
19300
19301// FastEmit functions for X86ISD::VCVT2PH2HF8.
19302
19303Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19304 if (RetVT.SimpleTy != MVT::v16i8)
19305 return Register();
19306 if ((Subtarget->hasAVX10_2())) {
19307 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19308 }
19309 return Register();
19310}
19311
19312Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19313 if (RetVT.SimpleTy != MVT::v32i8)
19314 return Register();
19315 if ((Subtarget->hasAVX10_2())) {
19316 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19317 }
19318 return Register();
19319}
19320
19321Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19322 if (RetVT.SimpleTy != MVT::v64i8)
19323 return Register();
19324 if ((Subtarget->hasAVX10_2())) {
19325 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19326 }
19327 return Register();
19328}
19329
19330Register fastEmit_X86ISD_VCVT2PH2HF8_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19331 switch (VT.SimpleTy) {
19332 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v8f16_rr(RetVT, Op0, Op1);
19333 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v16f16_rr(RetVT, Op0, Op1);
19334 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v32f16_rr(RetVT, Op0, Op1);
19335 default: return Register();
19336 }
19337}
19338
19339// FastEmit functions for X86ISD::VCVT2PH2HF8S.
19340
19341Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19342 if (RetVT.SimpleTy != MVT::v16i8)
19343 return Register();
19344 if ((Subtarget->hasAVX10_2())) {
19345 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19346 }
19347 return Register();
19348}
19349
19350Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19351 if (RetVT.SimpleTy != MVT::v32i8)
19352 return Register();
19353 if ((Subtarget->hasAVX10_2())) {
19354 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19355 }
19356 return Register();
19357}
19358
19359Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19360 if (RetVT.SimpleTy != MVT::v64i8)
19361 return Register();
19362 if ((Subtarget->hasAVX10_2())) {
19363 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZrr, RC: &X86::VR512RegClass, Op0, Op1);
19364 }
19365 return Register();
19366}
19367
19368Register fastEmit_X86ISD_VCVT2PH2HF8S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19369 switch (VT.SimpleTy) {
19370 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v8f16_rr(RetVT, Op0, Op1);
19371 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v16f16_rr(RetVT, Op0, Op1);
19372 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v32f16_rr(RetVT, Op0, Op1);
19373 default: return Register();
19374 }
19375}
19376
19377// FastEmit functions for X86ISD::VFCMULC.
19378
19379Register fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19380 if (RetVT.SimpleTy != MVT::v4f32)
19381 return Register();
19382 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19383 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19384 }
19385 return Register();
19386}
19387
19388Register fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19389 if (RetVT.SimpleTy != MVT::v8f32)
19390 return Register();
19391 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19392 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19393 }
19394 return Register();
19395}
19396
19397Register fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19398 if (RetVT.SimpleTy != MVT::v16f32)
19399 return Register();
19400 if ((Subtarget->hasFP16())) {
19401 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
19402 }
19403 return Register();
19404}
19405
19406Register fastEmit_X86ISD_VFCMULC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19407 switch (VT.SimpleTy) {
19408 case MVT::v4f32: return fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
19409 case MVT::v8f32: return fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
19410 case MVT::v16f32: return fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
19411 default: return Register();
19412 }
19413}
19414
19415// FastEmit functions for X86ISD::VFCMULCSH.
19416
19417Register fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19418 if (RetVT.SimpleTy != MVT::v4f32)
19419 return Register();
19420 if ((Subtarget->hasFP16())) {
19421 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
19422 }
19423 return Register();
19424}
19425
19426Register fastEmit_X86ISD_VFCMULCSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19427 switch (VT.SimpleTy) {
19428 case MVT::v4f32: return fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
19429 default: return Register();
19430 }
19431}
19432
19433// FastEmit functions for X86ISD::VFMULC.
19434
19435Register fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19436 if (RetVT.SimpleTy != MVT::v4f32)
19437 return Register();
19438 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19439 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19440 }
19441 return Register();
19442}
19443
19444Register fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19445 if (RetVT.SimpleTy != MVT::v8f32)
19446 return Register();
19447 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19448 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19449 }
19450 return Register();
19451}
19452
19453Register fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19454 if (RetVT.SimpleTy != MVT::v16f32)
19455 return Register();
19456 if ((Subtarget->hasFP16())) {
19457 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
19458 }
19459 return Register();
19460}
19461
19462Register fastEmit_X86ISD_VFMULC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19463 switch (VT.SimpleTy) {
19464 case MVT::v4f32: return fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
19465 case MVT::v8f32: return fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
19466 case MVT::v16f32: return fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
19467 default: return Register();
19468 }
19469}
19470
19471// FastEmit functions for X86ISD::VFMULCSH.
19472
19473Register fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19474 if (RetVT.SimpleTy != MVT::v4f32)
19475 return Register();
19476 if ((Subtarget->hasFP16())) {
19477 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
19478 }
19479 return Register();
19480}
19481
19482Register fastEmit_X86ISD_VFMULCSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19483 switch (VT.SimpleTy) {
19484 case MVT::v4f32: return fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
19485 default: return Register();
19486 }
19487}
19488
19489// FastEmit functions for X86ISD::VFPROUND2.
19490
19491Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8f16_rr(Register Op0, Register Op1) {
19492 if ((Subtarget->hasAVX10_2())) {
19493 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19494 }
19495 return Register();
19496}
19497
19498Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8bf16_rr(Register Op0, Register Op1) {
19499 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
19500 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19501 }
19502 return Register();
19503}
19504
19505Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19506switch (RetVT.SimpleTy) {
19507 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8f16_rr(Op0, Op1);
19508 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8bf16_rr(Op0, Op1);
19509 default: return Register();
19510}
19511}
19512
19513Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16f16_rr(Register Op0, Register Op1) {
19514 if ((Subtarget->hasAVX10_2())) {
19515 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19516 }
19517 return Register();
19518}
19519
19520Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16bf16_rr(Register Op0, Register Op1) {
19521 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
19522 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19523 }
19524 return Register();
19525}
19526
19527Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19528switch (RetVT.SimpleTy) {
19529 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16f16_rr(Op0, Op1);
19530 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16bf16_rr(Op0, Op1);
19531 default: return Register();
19532}
19533}
19534
19535Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32f16_rr(Register Op0, Register Op1) {
19536 if ((Subtarget->hasAVX10_2())) {
19537 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZrr, RC: &X86::VR512RegClass, Op0, Op1);
19538 }
19539 return Register();
19540}
19541
19542Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32bf16_rr(Register Op0, Register Op1) {
19543 if ((Subtarget->hasBF16())) {
19544 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19545 }
19546 return Register();
19547}
19548
19549Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19550switch (RetVT.SimpleTy) {
19551 case MVT::v32f16: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32f16_rr(Op0, Op1);
19552 case MVT::v32bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32bf16_rr(Op0, Op1);
19553 default: return Register();
19554}
19555}
19556
19557Register fastEmit_X86ISD_VFPROUND2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19558 switch (VT.SimpleTy) {
19559 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_rr(RetVT, Op0, Op1);
19560 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_rr(RetVT, Op0, Op1);
19561 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_rr(RetVT, Op0, Op1);
19562 default: return Register();
19563 }
19564}
19565
19566// FastEmit functions for X86ISD::VP2INTERSECT.
19567
19568Register fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19569 if (RetVT.SimpleTy != MVT::Untyped)
19570 return Register();
19571 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19572 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ128rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
19573 }
19574 return Register();
19575}
19576
19577Register fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19578 if (RetVT.SimpleTy != MVT::Untyped)
19579 return Register();
19580 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19581 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ256rr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
19582 }
19583 return Register();
19584}
19585
19586Register fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19587 if (RetVT.SimpleTy != MVT::Untyped)
19588 return Register();
19589 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
19590 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZrr, RC: &X86::VK16PAIRRegClass, Op0, Op1);
19591 }
19592 return Register();
19593}
19594
19595Register fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19596 if (RetVT.SimpleTy != MVT::Untyped)
19597 return Register();
19598 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19599 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ128rr, RC: &X86::VK2PAIRRegClass, Op0, Op1);
19600 }
19601 return Register();
19602}
19603
19604Register fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19605 if (RetVT.SimpleTy != MVT::Untyped)
19606 return Register();
19607 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19608 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ256rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
19609 }
19610 return Register();
19611}
19612
19613Register fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19614 if (RetVT.SimpleTy != MVT::Untyped)
19615 return Register();
19616 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
19617 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZrr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
19618 }
19619 return Register();
19620}
19621
19622Register fastEmit_X86ISD_VP2INTERSECT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19623 switch (VT.SimpleTy) {
19624 case MVT::v4i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(RetVT, Op0, Op1);
19625 case MVT::v8i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(RetVT, Op0, Op1);
19626 case MVT::v16i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(RetVT, Op0, Op1);
19627 case MVT::v2i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(RetVT, Op0, Op1);
19628 case MVT::v4i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(RetVT, Op0, Op1);
19629 case MVT::v8i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(RetVT, Op0, Op1);
19630 default: return Register();
19631 }
19632}
19633
19634// FastEmit functions for X86ISD::VPERMV.
19635
19636Register fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19637 if (RetVT.SimpleTy != MVT::v16i8)
19638 return Register();
19639 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
19640 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19641 }
19642 return Register();
19643}
19644
19645Register fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19646 if (RetVT.SimpleTy != MVT::v32i8)
19647 return Register();
19648 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
19649 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19650 }
19651 return Register();
19652}
19653
19654Register fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19655 if (RetVT.SimpleTy != MVT::v64i8)
19656 return Register();
19657 if ((Subtarget->hasVBMI())) {
19658 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZrr, RC: &X86::VR512RegClass, Op0, Op1);
19659 }
19660 return Register();
19661}
19662
19663Register fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19664 if (RetVT.SimpleTy != MVT::v8i16)
19665 return Register();
19666 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19667 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19668 }
19669 return Register();
19670}
19671
19672Register fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19673 if (RetVT.SimpleTy != MVT::v16i16)
19674 return Register();
19675 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19676 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19677 }
19678 return Register();
19679}
19680
19681Register fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19682 if (RetVT.SimpleTy != MVT::v32i16)
19683 return Register();
19684 if ((Subtarget->hasBWI())) {
19685 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19686 }
19687 return Register();
19688}
19689
19690Register fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19691 if (RetVT.SimpleTy != MVT::v8i32)
19692 return Register();
19693 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19694 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19695 }
19696 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19697 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19698 }
19699 return Register();
19700}
19701
19702Register fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19703 if (RetVT.SimpleTy != MVT::v16i32)
19704 return Register();
19705 if ((Subtarget->hasAVX512())) {
19706 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19707 }
19708 return Register();
19709}
19710
19711Register fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19712 if (RetVT.SimpleTy != MVT::v4i64)
19713 return Register();
19714 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19715 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19716 }
19717 return Register();
19718}
19719
19720Register fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19721 if (RetVT.SimpleTy != MVT::v8i64)
19722 return Register();
19723 if ((Subtarget->hasAVX512())) {
19724 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19725 }
19726 return Register();
19727}
19728
19729Register fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19730 switch (VT.SimpleTy) {
19731 case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op1);
19732 case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op1);
19733 case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op1);
19734 case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op1);
19735 case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op1);
19736 case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op1);
19737 case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op1);
19738 case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op1);
19739 case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op1);
19740 case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op1);
19741 default: return Register();
19742 }
19743}
19744
19745// FastEmit functions for X86ISD::VPMADDUBSW.
19746
19747Register fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19748 if (RetVT.SimpleTy != MVT::v8i16)
19749 return Register();
19750 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19751 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19752 }
19753 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
19754 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
19755 }
19756 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19757 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
19758 }
19759 return Register();
19760}
19761
19762Register fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19763 if (RetVT.SimpleTy != MVT::v16i16)
19764 return Register();
19765 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19766 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19767 }
19768 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19769 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
19770 }
19771 return Register();
19772}
19773
19774Register fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19775 if (RetVT.SimpleTy != MVT::v32i16)
19776 return Register();
19777 if ((Subtarget->hasBWI())) {
19778 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19779 }
19780 return Register();
19781}
19782
19783Register fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19784 switch (VT.SimpleTy) {
19785 case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op1);
19786 case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op1);
19787 case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op1);
19788 default: return Register();
19789 }
19790}
19791
19792// FastEmit functions for X86ISD::VPMADDWD.
19793
19794Register fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19795 if (RetVT.SimpleTy != MVT::v4i32)
19796 return Register();
19797 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19798 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19799 }
19800 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19801 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19802 }
19803 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19804 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19805 }
19806 return Register();
19807}
19808
19809Register fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19810 if (RetVT.SimpleTy != MVT::v8i32)
19811 return Register();
19812 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19813 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19814 }
19815 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19816 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19817 }
19818 return Register();
19819}
19820
19821Register fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19822 if (RetVT.SimpleTy != MVT::v16i32)
19823 return Register();
19824 if ((Subtarget->hasBWI())) {
19825 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19826 }
19827 return Register();
19828}
19829
19830Register fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19831 switch (VT.SimpleTy) {
19832 case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op1);
19833 case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op1);
19834 case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op1);
19835 default: return Register();
19836 }
19837}
19838
19839// FastEmit functions for X86ISD::VPSHA.
19840
19841Register fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19842 if (RetVT.SimpleTy != MVT::v16i8)
19843 return Register();
19844 if ((Subtarget->hasXOP())) {
19845 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHABrr, RC: &X86::VR128RegClass, Op0, Op1);
19846 }
19847 return Register();
19848}
19849
19850Register fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19851 if (RetVT.SimpleTy != MVT::v8i16)
19852 return Register();
19853 if ((Subtarget->hasXOP())) {
19854 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAWrr, RC: &X86::VR128RegClass, Op0, Op1);
19855 }
19856 return Register();
19857}
19858
19859Register fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19860 if (RetVT.SimpleTy != MVT::v4i32)
19861 return Register();
19862 if ((Subtarget->hasXOP())) {
19863 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHADrr, RC: &X86::VR128RegClass, Op0, Op1);
19864 }
19865 return Register();
19866}
19867
19868Register fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19869 if (RetVT.SimpleTy != MVT::v2i64)
19870 return Register();
19871 if ((Subtarget->hasXOP())) {
19872 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAQrr, RC: &X86::VR128RegClass, Op0, Op1);
19873 }
19874 return Register();
19875}
19876
19877Register fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19878 switch (VT.SimpleTy) {
19879 case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op1);
19880 case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op1);
19881 case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op1);
19882 case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op1);
19883 default: return Register();
19884 }
19885}
19886
19887// FastEmit functions for X86ISD::VPSHL.
19888
19889Register fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19890 if (RetVT.SimpleTy != MVT::v16i8)
19891 return Register();
19892 if ((Subtarget->hasXOP())) {
19893 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLBrr, RC: &X86::VR128RegClass, Op0, Op1);
19894 }
19895 return Register();
19896}
19897
19898Register fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19899 if (RetVT.SimpleTy != MVT::v8i16)
19900 return Register();
19901 if ((Subtarget->hasXOP())) {
19902 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19903 }
19904 return Register();
19905}
19906
19907Register fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19908 if (RetVT.SimpleTy != MVT::v4i32)
19909 return Register();
19910 if ((Subtarget->hasXOP())) {
19911 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19912 }
19913 return Register();
19914}
19915
19916Register fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19917 if (RetVT.SimpleTy != MVT::v2i64)
19918 return Register();
19919 if ((Subtarget->hasXOP())) {
19920 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLQrr, RC: &X86::VR128RegClass, Op0, Op1);
19921 }
19922 return Register();
19923}
19924
19925Register fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19926 switch (VT.SimpleTy) {
19927 case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op1);
19928 case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
19929 case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
19930 case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
19931 default: return Register();
19932 }
19933}
19934
19935// FastEmit functions for X86ISD::VPSHUFBITQMB.
19936
19937Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19938 if (RetVT.SimpleTy != MVT::v16i1)
19939 return Register();
19940 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
19941 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ128rr, RC: &X86::VK16RegClass, Op0, Op1);
19942 }
19943 return Register();
19944}
19945
19946Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19947 if (RetVT.SimpleTy != MVT::v32i1)
19948 return Register();
19949 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
19950 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ256rr, RC: &X86::VK32RegClass, Op0, Op1);
19951 }
19952 return Register();
19953}
19954
19955Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19956 if (RetVT.SimpleTy != MVT::v64i1)
19957 return Register();
19958 if ((Subtarget->hasBITALG())) {
19959 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZrr, RC: &X86::VK64RegClass, Op0, Op1);
19960 }
19961 return Register();
19962}
19963
19964Register fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19965 switch (VT.SimpleTy) {
19966 case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op1);
19967 case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op1);
19968 case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op1);
19969 default: return Register();
19970 }
19971}
19972
19973// FastEmit functions for X86ISD::VSHL.
19974
19975Register fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19976 if (RetVT.SimpleTy != MVT::v8i16)
19977 return Register();
19978 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19979 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19980 }
19981 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19982 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19983 }
19984 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19985 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
19986 }
19987 return Register();
19988}
19989
19990Register fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19991 if (RetVT.SimpleTy != MVT::v4i32)
19992 return Register();
19993 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19994 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19995 }
19996 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19997 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
19998 }
19999 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20000 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20001 }
20002 return Register();
20003}
20004
20005Register fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20006 if (RetVT.SimpleTy != MVT::v2i64)
20007 return Register();
20008 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20009 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20010 }
20011 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20012 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20013 }
20014 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20015 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20016 }
20017 return Register();
20018}
20019
20020Register fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20021 switch (VT.SimpleTy) {
20022 case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
20023 case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
20024 case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
20025 default: return Register();
20026 }
20027}
20028
20029// FastEmit functions for X86ISD::VSHLV.
20030
20031Register fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20032 if (RetVT.SimpleTy != MVT::v8i16)
20033 return Register();
20034 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20035 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20036 }
20037 return Register();
20038}
20039
20040Register fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20041 if (RetVT.SimpleTy != MVT::v16i16)
20042 return Register();
20043 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20044 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20045 }
20046 return Register();
20047}
20048
20049Register fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20050 if (RetVT.SimpleTy != MVT::v32i16)
20051 return Register();
20052 if ((Subtarget->hasBWI())) {
20053 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20054 }
20055 return Register();
20056}
20057
20058Register fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20059 if (RetVT.SimpleTy != MVT::v4i32)
20060 return Register();
20061 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20062 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20063 }
20064 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20065 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20066 }
20067 return Register();
20068}
20069
20070Register fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20071 if (RetVT.SimpleTy != MVT::v8i32)
20072 return Register();
20073 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20074 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20075 }
20076 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20077 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20078 }
20079 return Register();
20080}
20081
20082Register fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20083 if (RetVT.SimpleTy != MVT::v16i32)
20084 return Register();
20085 if ((Subtarget->hasAVX512())) {
20086 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20087 }
20088 return Register();
20089}
20090
20091Register fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20092 if (RetVT.SimpleTy != MVT::v2i64)
20093 return Register();
20094 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20095 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20096 }
20097 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20098 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
20099 }
20100 return Register();
20101}
20102
20103Register fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20104 if (RetVT.SimpleTy != MVT::v4i64)
20105 return Register();
20106 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20107 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20108 }
20109 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20110 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
20111 }
20112 return Register();
20113}
20114
20115Register fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20116 if (RetVT.SimpleTy != MVT::v8i64)
20117 return Register();
20118 if ((Subtarget->hasAVX512())) {
20119 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20120 }
20121 return Register();
20122}
20123
20124Register fastEmit_X86ISD_VSHLV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20125 switch (VT.SimpleTy) {
20126 case MVT::v8i16: return fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(RetVT, Op0, Op1);
20127 case MVT::v16i16: return fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(RetVT, Op0, Op1);
20128 case MVT::v32i16: return fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(RetVT, Op0, Op1);
20129 case MVT::v4i32: return fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(RetVT, Op0, Op1);
20130 case MVT::v8i32: return fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(RetVT, Op0, Op1);
20131 case MVT::v16i32: return fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(RetVT, Op0, Op1);
20132 case MVT::v2i64: return fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(RetVT, Op0, Op1);
20133 case MVT::v4i64: return fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(RetVT, Op0, Op1);
20134 case MVT::v8i64: return fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(RetVT, Op0, Op1);
20135 default: return Register();
20136 }
20137}
20138
20139// FastEmit functions for X86ISD::VSRA.
20140
20141Register fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20142 if (RetVT.SimpleTy != MVT::v8i16)
20143 return Register();
20144 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20145 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20146 }
20147 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20148 return fastEmitInst_rr(MachineInstOpcode: X86::PSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
20149 }
20150 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
20151 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
20152 }
20153 return Register();
20154}
20155
20156Register fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20157 if (RetVT.SimpleTy != MVT::v4i32)
20158 return Register();
20159 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20160 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20161 }
20162 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20163 return fastEmitInst_rr(MachineInstOpcode: X86::PSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
20164 }
20165 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20166 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
20167 }
20168 return Register();
20169}
20170
20171Register fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20172 if (RetVT.SimpleTy != MVT::v2i64)
20173 return Register();
20174 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20175 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20176 }
20177 return Register();
20178}
20179
20180Register fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20181 switch (VT.SimpleTy) {
20182 case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op1);
20183 case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op1);
20184 case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op1);
20185 default: return Register();
20186 }
20187}
20188
20189// FastEmit functions for X86ISD::VSRAV.
20190
20191Register fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20192 if (RetVT.SimpleTy != MVT::v8i16)
20193 return Register();
20194 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20195 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20196 }
20197 return Register();
20198}
20199
20200Register fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20201 if (RetVT.SimpleTy != MVT::v16i16)
20202 return Register();
20203 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20204 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20205 }
20206 return Register();
20207}
20208
20209Register fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20210 if (RetVT.SimpleTy != MVT::v32i16)
20211 return Register();
20212 if ((Subtarget->hasBWI())) {
20213 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20214 }
20215 return Register();
20216}
20217
20218Register fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20219 if (RetVT.SimpleTy != MVT::v4i32)
20220 return Register();
20221 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20222 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20223 }
20224 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20225 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20226 }
20227 return Register();
20228}
20229
20230Register fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20231 if (RetVT.SimpleTy != MVT::v8i32)
20232 return Register();
20233 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20234 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20235 }
20236 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20237 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20238 }
20239 return Register();
20240}
20241
20242Register fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20243 if (RetVT.SimpleTy != MVT::v16i32)
20244 return Register();
20245 if ((Subtarget->hasAVX512())) {
20246 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20247 }
20248 return Register();
20249}
20250
20251Register fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20252 if (RetVT.SimpleTy != MVT::v2i64)
20253 return Register();
20254 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20255 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20256 }
20257 return Register();
20258}
20259
20260Register fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20261 if (RetVT.SimpleTy != MVT::v4i64)
20262 return Register();
20263 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20264 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20265 }
20266 return Register();
20267}
20268
20269Register fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20270 if (RetVT.SimpleTy != MVT::v8i64)
20271 return Register();
20272 if ((Subtarget->hasAVX512())) {
20273 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20274 }
20275 return Register();
20276}
20277
20278Register fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20279 switch (VT.SimpleTy) {
20280 case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op1);
20281 case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op1);
20282 case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op1);
20283 case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op1);
20284 case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op1);
20285 case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op1);
20286 case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op1);
20287 case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op1);
20288 case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op1);
20289 default: return Register();
20290 }
20291}
20292
20293// FastEmit functions for X86ISD::VSRL.
20294
20295Register fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20296 if (RetVT.SimpleTy != MVT::v8i16)
20297 return Register();
20298 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20299 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20300 }
20301 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20302 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20303 }
20304 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
20305 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20306 }
20307 return Register();
20308}
20309
20310Register fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20311 if (RetVT.SimpleTy != MVT::v4i32)
20312 return Register();
20313 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20314 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20315 }
20316 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20317 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20318 }
20319 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20320 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20321 }
20322 return Register();
20323}
20324
20325Register fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20326 if (RetVT.SimpleTy != MVT::v2i64)
20327 return Register();
20328 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20329 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20330 }
20331 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20332 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20333 }
20334 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20335 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20336 }
20337 return Register();
20338}
20339
20340Register fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20341 switch (VT.SimpleTy) {
20342 case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op1);
20343 case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op1);
20344 case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op1);
20345 default: return Register();
20346 }
20347}
20348
20349// FastEmit functions for X86ISD::VSRLV.
20350
20351Register fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20352 if (RetVT.SimpleTy != MVT::v8i16)
20353 return Register();
20354 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20355 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20356 }
20357 return Register();
20358}
20359
20360Register fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20361 if (RetVT.SimpleTy != MVT::v16i16)
20362 return Register();
20363 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20364 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20365 }
20366 return Register();
20367}
20368
20369Register fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20370 if (RetVT.SimpleTy != MVT::v32i16)
20371 return Register();
20372 if ((Subtarget->hasBWI())) {
20373 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20374 }
20375 return Register();
20376}
20377
20378Register fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20379 if (RetVT.SimpleTy != MVT::v4i32)
20380 return Register();
20381 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20382 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20383 }
20384 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20385 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20386 }
20387 return Register();
20388}
20389
20390Register fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20391 if (RetVT.SimpleTy != MVT::v8i32)
20392 return Register();
20393 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20394 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20395 }
20396 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20397 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20398 }
20399 return Register();
20400}
20401
20402Register fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20403 if (RetVT.SimpleTy != MVT::v16i32)
20404 return Register();
20405 if ((Subtarget->hasAVX512())) {
20406 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20407 }
20408 return Register();
20409}
20410
20411Register fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20412 if (RetVT.SimpleTy != MVT::v2i64)
20413 return Register();
20414 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20415 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20416 }
20417 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20418 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
20419 }
20420 return Register();
20421}
20422
20423Register fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20424 if (RetVT.SimpleTy != MVT::v4i64)
20425 return Register();
20426 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20427 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20428 }
20429 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20430 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
20431 }
20432 return Register();
20433}
20434
20435Register fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20436 if (RetVT.SimpleTy != MVT::v8i64)
20437 return Register();
20438 if ((Subtarget->hasAVX512())) {
20439 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20440 }
20441 return Register();
20442}
20443
20444Register fastEmit_X86ISD_VSRLV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20445 switch (VT.SimpleTy) {
20446 case MVT::v8i16: return fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(RetVT, Op0, Op1);
20447 case MVT::v16i16: return fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(RetVT, Op0, Op1);
20448 case MVT::v32i16: return fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(RetVT, Op0, Op1);
20449 case MVT::v4i32: return fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(RetVT, Op0, Op1);
20450 case MVT::v8i32: return fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(RetVT, Op0, Op1);
20451 case MVT::v16i32: return fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(RetVT, Op0, Op1);
20452 case MVT::v2i64: return fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(RetVT, Op0, Op1);
20453 case MVT::v4i64: return fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(RetVT, Op0, Op1);
20454 case MVT::v8i64: return fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(RetVT, Op0, Op1);
20455 default: return Register();
20456 }
20457}
20458
20459// Top-level FastEmit function.
20460
20461Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
20462 switch (Opcode) {
20463 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
20464 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
20465 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
20466 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
20467 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
20468 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
20469 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
20470 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
20471 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
20472 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
20473 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
20474 case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op1);
20475 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
20476 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
20477 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
20478 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
20479 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
20480 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
20481 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
20482 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
20483 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
20484 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
20485 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
20486 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
20487 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
20488 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
20489 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
20490 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
20491 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
20492 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
20493 case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op1);
20494 case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op1);
20495 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op1);
20496 case X86ISD::BT: return fastEmit_X86ISD_BT_rr(VT, RetVT, Op0, Op1);
20497 case X86ISD::BZHI: return fastEmit_X86ISD_BZHI_rr(VT, RetVT, Op0, Op1);
20498 case X86ISD::CMP: return fastEmit_X86ISD_CMP_rr(VT, RetVT, Op0, Op1);
20499 case X86ISD::COMI: return fastEmit_X86ISD_COMI_rr(VT, RetVT, Op0, Op1);
20500 case X86ISD::COMX: return fastEmit_X86ISD_COMX_rr(VT, RetVT, Op0, Op1);
20501 case X86ISD::FADDS: return fastEmit_X86ISD_FADDS_rr(VT, RetVT, Op0, Op1);
20502 case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op1);
20503 case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op1);
20504 case X86ISD::FCMP: return fastEmit_X86ISD_FCMP_rr(VT, RetVT, Op0, Op1);
20505 case X86ISD::FDIVS: return fastEmit_X86ISD_FDIVS_rr(VT, RetVT, Op0, Op1);
20506 case X86ISD::FGETEXPS: return fastEmit_X86ISD_FGETEXPS_rr(VT, RetVT, Op0, Op1);
20507 case X86ISD::FGETEXPS_SAE: return fastEmit_X86ISD_FGETEXPS_SAE_rr(VT, RetVT, Op0, Op1);
20508 case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op1);
20509 case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op1);
20510 case X86ISD::FMAX: return fastEmit_X86ISD_FMAX_rr(VT, RetVT, Op0, Op1);
20511 case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op1);
20512 case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op1);
20513 case X86ISD::FMAXS_SAE: return fastEmit_X86ISD_FMAXS_SAE_rr(VT, RetVT, Op0, Op1);
20514 case X86ISD::FMAX_SAE: return fastEmit_X86ISD_FMAX_SAE_rr(VT, RetVT, Op0, Op1);
20515 case X86ISD::FMIN: return fastEmit_X86ISD_FMIN_rr(VT, RetVT, Op0, Op1);
20516 case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op1);
20517 case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op1);
20518 case X86ISD::FMINS_SAE: return fastEmit_X86ISD_FMINS_SAE_rr(VT, RetVT, Op0, Op1);
20519 case X86ISD::FMIN_SAE: return fastEmit_X86ISD_FMIN_SAE_rr(VT, RetVT, Op0, Op1);
20520 case X86ISD::FMULS: return fastEmit_X86ISD_FMULS_rr(VT, RetVT, Op0, Op1);
20521 case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op1);
20522 case X86ISD::FP80_ADD: return fastEmit_X86ISD_FP80_ADD_rr(VT, RetVT, Op0, Op1);
20523 case X86ISD::FSQRTS: return fastEmit_X86ISD_FSQRTS_rr(VT, RetVT, Op0, Op1);
20524 case X86ISD::FSUBS: return fastEmit_X86ISD_FSUBS_rr(VT, RetVT, Op0, Op1);
20525 case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op1);
20526 case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op1);
20527 case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op1);
20528 case X86ISD::HADDS: return fastEmit_X86ISD_HADDS_rr(VT, RetVT, Op0, Op1);
20529 case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op1);
20530 case X86ISD::HSUBS: return fastEmit_X86ISD_HSUBS_rr(VT, RetVT, Op0, Op1);
20531 case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op1);
20532 case X86ISD::KORTEST: return fastEmit_X86ISD_KORTEST_rr(VT, RetVT, Op0, Op1);
20533 case X86ISD::KTEST: return fastEmit_X86ISD_KTEST_rr(VT, RetVT, Op0, Op1);
20534 case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op1);
20535 case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op1);
20536 case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op1);
20537 case X86ISD::MOVSH: return fastEmit_X86ISD_MOVSH_rr(VT, RetVT, Op0, Op1);
20538 case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op1);
20539 case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op1);
20540 case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op1);
20541 case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op1);
20542 case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op1);
20543 case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op1);
20544 case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op1);
20545 case X86ISD::PDEP: return fastEmit_X86ISD_PDEP_rr(VT, RetVT, Op0, Op1);
20546 case X86ISD::PEXT: return fastEmit_X86ISD_PEXT_rr(VT, RetVT, Op0, Op1);
20547 case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op1);
20548 case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op1);
20549 case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op1);
20550 case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op1);
20551 case X86ISD::PTEST: return fastEmit_X86ISD_PTEST_rr(VT, RetVT, Op0, Op1);
20552 case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op1);
20553 case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op1);
20554 case X86ISD::SCALEF: return fastEmit_X86ISD_SCALEF_rr(VT, RetVT, Op0, Op1);
20555 case X86ISD::SCALEFS: return fastEmit_X86ISD_SCALEFS_rr(VT, RetVT, Op0, Op1);
20556 case X86ISD::STRICT_FCMP: return fastEmit_X86ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
20557 case X86ISD::STRICT_FCMPS: return fastEmit_X86ISD_STRICT_FCMPS_rr(VT, RetVT, Op0, Op1);
20558 case X86ISD::STRICT_FMAX: return fastEmit_X86ISD_STRICT_FMAX_rr(VT, RetVT, Op0, Op1);
20559 case X86ISD::STRICT_FMIN: return fastEmit_X86ISD_STRICT_FMIN_rr(VT, RetVT, Op0, Op1);
20560 case X86ISD::STRICT_FP80_ADD: return fastEmit_X86ISD_STRICT_FP80_ADD_rr(VT, RetVT, Op0, Op1);
20561 case X86ISD::TESTP: return fastEmit_X86ISD_TESTP_rr(VT, RetVT, Op0, Op1);
20562 case X86ISD::UCOMI: return fastEmit_X86ISD_UCOMI_rr(VT, RetVT, Op0, Op1);
20563 case X86ISD::UCOMX: return fastEmit_X86ISD_UCOMX_rr(VT, RetVT, Op0, Op1);
20564 case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op1);
20565 case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op1);
20566 case X86ISD::VCVT2PH2BF8: return fastEmit_X86ISD_VCVT2PH2BF8_rr(VT, RetVT, Op0, Op1);
20567 case X86ISD::VCVT2PH2BF8S: return fastEmit_X86ISD_VCVT2PH2BF8S_rr(VT, RetVT, Op0, Op1);
20568 case X86ISD::VCVT2PH2HF8: return fastEmit_X86ISD_VCVT2PH2HF8_rr(VT, RetVT, Op0, Op1);
20569 case X86ISD::VCVT2PH2HF8S: return fastEmit_X86ISD_VCVT2PH2HF8S_rr(VT, RetVT, Op0, Op1);
20570 case X86ISD::VFCMULC: return fastEmit_X86ISD_VFCMULC_rr(VT, RetVT, Op0, Op1);
20571 case X86ISD::VFCMULCSH: return fastEmit_X86ISD_VFCMULCSH_rr(VT, RetVT, Op0, Op1);
20572 case X86ISD::VFMULC: return fastEmit_X86ISD_VFMULC_rr(VT, RetVT, Op0, Op1);
20573 case X86ISD::VFMULCSH: return fastEmit_X86ISD_VFMULCSH_rr(VT, RetVT, Op0, Op1);
20574 case X86ISD::VFPROUND2: return fastEmit_X86ISD_VFPROUND2_rr(VT, RetVT, Op0, Op1);
20575 case X86ISD::VP2INTERSECT: return fastEmit_X86ISD_VP2INTERSECT_rr(VT, RetVT, Op0, Op1);
20576 case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op1);
20577 case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op1);
20578 case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op1);
20579 case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op1);
20580 case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op1);
20581 case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op1);
20582 case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op1);
20583 case X86ISD::VSHLV: return fastEmit_X86ISD_VSHLV_rr(VT, RetVT, Op0, Op1);
20584 case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op1);
20585 case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op1);
20586 case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op1);
20587 case X86ISD::VSRLV: return fastEmit_X86ISD_VSRLV_rr(VT, RetVT, Op0, Op1);
20588 default: return Register();
20589 }
20590}
20591
20592// FastEmit functions for ISD::ADD.
20593
20594Register fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20595 if (RetVT.SimpleTy != MVT::i8)
20596 return Register();
20597 if ((Subtarget->hasNDD())) {
20598 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20599 }
20600 if ((!Subtarget->hasNDD())) {
20601 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20602 }
20603 return Register();
20604}
20605
20606Register fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20607 if (RetVT.SimpleTy != MVT::i16)
20608 return Register();
20609 if ((Subtarget->hasNDD())) {
20610 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20611 }
20612 if ((!Subtarget->hasNDD())) {
20613 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20614 }
20615 return Register();
20616}
20617
20618Register fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20619 if (RetVT.SimpleTy != MVT::i32)
20620 return Register();
20621 if ((Subtarget->hasNDD())) {
20622 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20623 }
20624 if ((!Subtarget->hasNDD())) {
20625 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20626 }
20627 return Register();
20628}
20629
20630Register fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20631 switch (VT.SimpleTy) {
20632 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, imm1);
20633 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, imm1);
20634 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, imm1);
20635 default: return Register();
20636 }
20637}
20638
20639// FastEmit functions for ISD::AND.
20640
20641Register fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20642 if (RetVT.SimpleTy != MVT::i8)
20643 return Register();
20644 if ((Subtarget->hasNDD())) {
20645 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20646 }
20647 if ((!Subtarget->hasNDD())) {
20648 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20649 }
20650 return Register();
20651}
20652
20653Register fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20654 if (RetVT.SimpleTy != MVT::i16)
20655 return Register();
20656 if ((Subtarget->hasNDD())) {
20657 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20658 }
20659 if ((!Subtarget->hasNDD())) {
20660 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20661 }
20662 return Register();
20663}
20664
20665Register fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20666 if (RetVT.SimpleTy != MVT::i32)
20667 return Register();
20668 if ((Subtarget->hasNDD())) {
20669 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20670 }
20671 if ((!Subtarget->hasNDD())) {
20672 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20673 }
20674 return Register();
20675}
20676
20677Register fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20678 switch (VT.SimpleTy) {
20679 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, imm1);
20680 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, imm1);
20681 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, imm1);
20682 default: return Register();
20683 }
20684}
20685
20686// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
20687
20688Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20689 if (RetVT.SimpleTy != MVT::i32)
20690 return Register();
20691 if ((Subtarget->hasDQI())) {
20692 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDZrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20693 }
20694 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
20695 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRDrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20696 }
20697 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
20698 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20699 }
20700 return Register();
20701}
20702
20703Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20704 if (RetVT.SimpleTy != MVT::i64)
20705 return Register();
20706 if ((Subtarget->hasDQI())) {
20707 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQZrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20708 }
20709 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
20710 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRQrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20711 }
20712 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
20713 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20714 }
20715 return Register();
20716}
20717
20718Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20719 switch (VT.SimpleTy) {
20720 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, imm1);
20721 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, imm1);
20722 default: return Register();
20723 }
20724}
20725
20726// FastEmit functions for ISD::MUL.
20727
20728Register fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20729 if (RetVT.SimpleTy != MVT::i16)
20730 return Register();
20731 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL16rri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20732}
20733
20734Register fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20735 if (RetVT.SimpleTy != MVT::i32)
20736 return Register();
20737 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL32rri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20738}
20739
20740Register fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20741 switch (VT.SimpleTy) {
20742 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, imm1);
20743 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, imm1);
20744 default: return Register();
20745 }
20746}
20747
20748// FastEmit functions for ISD::OR.
20749
20750Register fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20751 if (RetVT.SimpleTy != MVT::i8)
20752 return Register();
20753 if ((Subtarget->hasNDD())) {
20754 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20755 }
20756 if ((!Subtarget->hasNDD())) {
20757 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20758 }
20759 return Register();
20760}
20761
20762Register fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20763 if (RetVT.SimpleTy != MVT::i16)
20764 return Register();
20765 if ((Subtarget->hasNDD())) {
20766 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20767 }
20768 if ((!Subtarget->hasNDD())) {
20769 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20770 }
20771 return Register();
20772}
20773
20774Register fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20775 if (RetVT.SimpleTy != MVT::i32)
20776 return Register();
20777 if ((Subtarget->hasNDD())) {
20778 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20779 }
20780 if ((!Subtarget->hasNDD())) {
20781 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20782 }
20783 return Register();
20784}
20785
20786Register fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20787 switch (VT.SimpleTy) {
20788 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, imm1);
20789 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, imm1);
20790 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, imm1);
20791 default: return Register();
20792 }
20793}
20794
20795// FastEmit functions for ISD::ROTL.
20796
20797Register fastEmit_ISD_ROTL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20798 if (RetVT.SimpleTy != MVT::i8)
20799 return Register();
20800 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20801 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20802 }
20803 if ((!Subtarget->hasNDD())) {
20804 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20805 }
20806 return Register();
20807}
20808
20809Register fastEmit_ISD_ROTL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20810 if (RetVT.SimpleTy != MVT::i16)
20811 return Register();
20812 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20813 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20814 }
20815 if ((!Subtarget->hasNDD())) {
20816 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20817 }
20818 return Register();
20819}
20820
20821Register fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20822 if (RetVT.SimpleTy != MVT::i32)
20823 return Register();
20824 if ((Subtarget->hasFastSHLDRotate())) {
20825 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20826 }
20827 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20828 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20829 }
20830 if ((!Subtarget->hasNDD())) {
20831 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20832 }
20833 return Register();
20834}
20835
20836Register fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20837 if (RetVT.SimpleTy != MVT::i64)
20838 return Register();
20839 if ((Subtarget->hasFastSHLDRotate())) {
20840 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20841 }
20842 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20843 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20844 }
20845 if ((!Subtarget->hasNDD())) {
20846 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20847 }
20848 return Register();
20849}
20850
20851Register fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20852 switch (VT.SimpleTy) {
20853 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_ri(RetVT, Op0, imm1);
20854 case MVT::i16: return fastEmit_ISD_ROTL_MVT_i16_ri(RetVT, Op0, imm1);
20855 case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, imm1);
20856 case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, imm1);
20857 default: return Register();
20858 }
20859}
20860
20861// FastEmit functions for ISD::ROTR.
20862
20863Register fastEmit_ISD_ROTR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20864 if (RetVT.SimpleTy != MVT::i8)
20865 return Register();
20866 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20867 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20868 }
20869 if ((!Subtarget->hasNDD())) {
20870 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20871 }
20872 return Register();
20873}
20874
20875Register fastEmit_ISD_ROTR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20876 if (RetVT.SimpleTy != MVT::i16)
20877 return Register();
20878 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20879 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20880 }
20881 if ((!Subtarget->hasNDD())) {
20882 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20883 }
20884 return Register();
20885}
20886
20887Register fastEmit_ISD_ROTR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20888 if (RetVT.SimpleTy != MVT::i32)
20889 return Register();
20890 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
20891 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri_EVEX, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20892 }
20893 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
20894 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20895 }
20896 if ((Subtarget->hasFastSHLDRotate())) {
20897 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20898 }
20899 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20900 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20901 }
20902 if ((!Subtarget->hasNDD())) {
20903 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20904 }
20905 return Register();
20906}
20907
20908Register fastEmit_ISD_ROTR_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20909 if (RetVT.SimpleTy != MVT::i64)
20910 return Register();
20911 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
20912 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri_EVEX, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20913 }
20914 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
20915 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20916 }
20917 if ((Subtarget->hasFastSHLDRotate())) {
20918 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20919 }
20920 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20921 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20922 }
20923 if ((!Subtarget->hasNDD())) {
20924 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20925 }
20926 return Register();
20927}
20928
20929Register fastEmit_ISD_ROTR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20930 switch (VT.SimpleTy) {
20931 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_ri(RetVT, Op0, imm1);
20932 case MVT::i16: return fastEmit_ISD_ROTR_MVT_i16_ri(RetVT, Op0, imm1);
20933 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri(RetVT, Op0, imm1);
20934 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_ri(RetVT, Op0, imm1);
20935 default: return Register();
20936 }
20937}
20938
20939// FastEmit functions for ISD::SHL.
20940
20941Register fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20942 if (RetVT.SimpleTy != MVT::i8)
20943 return Register();
20944 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20945 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20946 }
20947 if ((!Subtarget->hasNDD())) {
20948 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20949 }
20950 return Register();
20951}
20952
20953Register fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20954 if (RetVT.SimpleTy != MVT::i16)
20955 return Register();
20956 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20957 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20958 }
20959 if ((!Subtarget->hasNDD())) {
20960 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20961 }
20962 return Register();
20963}
20964
20965Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20966 if (RetVT.SimpleTy != MVT::i32)
20967 return Register();
20968 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20969 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20970 }
20971 if ((!Subtarget->hasNDD())) {
20972 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20973 }
20974 return Register();
20975}
20976
20977Register fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20978 if (RetVT.SimpleTy != MVT::i64)
20979 return Register();
20980 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20981 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20982 }
20983 if ((!Subtarget->hasNDD())) {
20984 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20985 }
20986 return Register();
20987}
20988
20989Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20990 switch (VT.SimpleTy) {
20991 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, imm1);
20992 case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, imm1);
20993 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
20994 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, imm1);
20995 default: return Register();
20996 }
20997}
20998
20999// FastEmit functions for ISD::SRA.
21000
21001Register fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21002 if (RetVT.SimpleTy != MVT::i8)
21003 return Register();
21004 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21005 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21006 }
21007 if ((!Subtarget->hasNDD())) {
21008 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21009 }
21010 return Register();
21011}
21012
21013Register fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21014 if (RetVT.SimpleTy != MVT::i16)
21015 return Register();
21016 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21017 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21018 }
21019 if ((!Subtarget->hasNDD())) {
21020 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21021 }
21022 return Register();
21023}
21024
21025Register fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21026 if (RetVT.SimpleTy != MVT::i32)
21027 return Register();
21028 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21029 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21030 }
21031 if ((!Subtarget->hasNDD())) {
21032 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21033 }
21034 return Register();
21035}
21036
21037Register fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21038 if (RetVT.SimpleTy != MVT::i64)
21039 return Register();
21040 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21041 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21042 }
21043 if ((!Subtarget->hasNDD())) {
21044 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21045 }
21046 return Register();
21047}
21048
21049Register fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21050 switch (VT.SimpleTy) {
21051 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, imm1);
21052 case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, imm1);
21053 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, imm1);
21054 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, imm1);
21055 default: return Register();
21056 }
21057}
21058
21059// FastEmit functions for ISD::SRL.
21060
21061Register fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21062 if (RetVT.SimpleTy != MVT::i8)
21063 return Register();
21064 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21065 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21066 }
21067 if ((!Subtarget->hasNDD())) {
21068 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21069 }
21070 return Register();
21071}
21072
21073Register fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21074 if (RetVT.SimpleTy != MVT::i16)
21075 return Register();
21076 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21077 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21078 }
21079 if ((!Subtarget->hasNDD())) {
21080 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21081 }
21082 return Register();
21083}
21084
21085Register fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21086 if (RetVT.SimpleTy != MVT::i32)
21087 return Register();
21088 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21089 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21090 }
21091 if ((!Subtarget->hasNDD())) {
21092 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21093 }
21094 return Register();
21095}
21096
21097Register fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21098 if (RetVT.SimpleTy != MVT::i64)
21099 return Register();
21100 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21101 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21102 }
21103 if ((!Subtarget->hasNDD())) {
21104 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21105 }
21106 return Register();
21107}
21108
21109Register fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21110 switch (VT.SimpleTy) {
21111 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, imm1);
21112 case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, imm1);
21113 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, imm1);
21114 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, imm1);
21115 default: return Register();
21116 }
21117}
21118
21119// FastEmit functions for ISD::SUB.
21120
21121Register fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21122 if (RetVT.SimpleTy != MVT::i8)
21123 return Register();
21124 if ((Subtarget->hasNDD())) {
21125 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21126 }
21127 if ((!Subtarget->hasNDD())) {
21128 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21129 }
21130 return Register();
21131}
21132
21133Register fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21134 if (RetVT.SimpleTy != MVT::i16)
21135 return Register();
21136 if ((Subtarget->hasNDD())) {
21137 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21138 }
21139 if ((!Subtarget->hasNDD())) {
21140 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21141 }
21142 return Register();
21143}
21144
21145Register fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21146 if (RetVT.SimpleTy != MVT::i32)
21147 return Register();
21148 if ((Subtarget->hasNDD())) {
21149 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21150 }
21151 if ((!Subtarget->hasNDD())) {
21152 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21153 }
21154 return Register();
21155}
21156
21157Register fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21158 switch (VT.SimpleTy) {
21159 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, imm1);
21160 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, imm1);
21161 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, imm1);
21162 default: return Register();
21163 }
21164}
21165
21166// FastEmit functions for ISD::XOR.
21167
21168Register fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21169 if (RetVT.SimpleTy != MVT::i8)
21170 return Register();
21171 if ((Subtarget->hasNDD())) {
21172 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21173 }
21174 if ((!Subtarget->hasNDD())) {
21175 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21176 }
21177 return Register();
21178}
21179
21180Register fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21181 if (RetVT.SimpleTy != MVT::i16)
21182 return Register();
21183 if ((Subtarget->hasNDD())) {
21184 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21185 }
21186 if ((!Subtarget->hasNDD())) {
21187 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21188 }
21189 return Register();
21190}
21191
21192Register fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21193 if (RetVT.SimpleTy != MVT::i32)
21194 return Register();
21195 if ((Subtarget->hasNDD())) {
21196 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21197 }
21198 if ((!Subtarget->hasNDD())) {
21199 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21200 }
21201 return Register();
21202}
21203
21204Register fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21205 switch (VT.SimpleTy) {
21206 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, imm1);
21207 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, imm1);
21208 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, imm1);
21209 default: return Register();
21210 }
21211}
21212
21213// FastEmit functions for X86ISD::BT.
21214
21215Register fastEmit_X86ISD_BT_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21216 if (RetVT.SimpleTy != MVT::i32)
21217 return Register();
21218 return fastEmitInst_ri(MachineInstOpcode: X86::BT16ri8, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21219}
21220
21221Register fastEmit_X86ISD_BT_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21222 if (RetVT.SimpleTy != MVT::i32)
21223 return Register();
21224 return fastEmitInst_ri(MachineInstOpcode: X86::BT32ri8, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21225}
21226
21227Register fastEmit_X86ISD_BT_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21228 if (RetVT.SimpleTy != MVT::i32)
21229 return Register();
21230 return fastEmitInst_ri(MachineInstOpcode: X86::BT64ri8, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21231}
21232
21233Register fastEmit_X86ISD_BT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21234 switch (VT.SimpleTy) {
21235 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_ri(RetVT, Op0, imm1);
21236 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_ri(RetVT, Op0, imm1);
21237 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_ri(RetVT, Op0, imm1);
21238 default: return Register();
21239 }
21240}
21241
21242// Top-level FastEmit function.
21243
21244Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
21245 if (VT == MVT::i64 && Predicate_i64immSExt32(Imm: imm1))
21246 if (Register Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, imm1))
21247 return Reg;
21248
21249 switch (Opcode) {
21250 case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, imm1);
21251 case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, imm1);
21252 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
21253 case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, imm1);
21254 case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, imm1);
21255 case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, imm1);
21256 case ISD::ROTR: return fastEmit_ISD_ROTR_ri(VT, RetVT, Op0, imm1);
21257 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
21258 case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, imm1);
21259 case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, imm1);
21260 case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, imm1);
21261 case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, imm1);
21262 case X86ISD::BT: return fastEmit_X86ISD_BT_ri(VT, RetVT, Op0, imm1);
21263 default: return Register();
21264 }
21265}
21266
21267// FastEmit functions for ISD::ADD.
21268
21269Register fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21270 if (RetVT.SimpleTy != MVT::i64)
21271 return Register();
21272 if ((Subtarget->hasNDD())) {
21273 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21274 }
21275 if ((!Subtarget->hasNDD())) {
21276 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21277 }
21278 return Register();
21279}
21280
21281Register fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21282 switch (VT.SimpleTy) {
21283 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21284 default: return Register();
21285 }
21286}
21287
21288// FastEmit functions for ISD::AND.
21289
21290Register fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21291 if (RetVT.SimpleTy != MVT::i64)
21292 return Register();
21293 if ((Subtarget->hasNDD())) {
21294 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21295 }
21296 if ((!Subtarget->hasNDD())) {
21297 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21298 }
21299 return Register();
21300}
21301
21302Register fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21303 switch (VT.SimpleTy) {
21304 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21305 default: return Register();
21306 }
21307}
21308
21309// FastEmit functions for ISD::MUL.
21310
21311Register fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21312 if (RetVT.SimpleTy != MVT::i64)
21313 return Register();
21314 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL64rri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21315}
21316
21317Register fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21318 switch (VT.SimpleTy) {
21319 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21320 default: return Register();
21321 }
21322}
21323
21324// FastEmit functions for ISD::OR.
21325
21326Register fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21327 if (RetVT.SimpleTy != MVT::i64)
21328 return Register();
21329 if ((Subtarget->hasNDD())) {
21330 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21331 }
21332 if ((!Subtarget->hasNDD())) {
21333 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21334 }
21335 return Register();
21336}
21337
21338Register fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21339 switch (VT.SimpleTy) {
21340 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21341 default: return Register();
21342 }
21343}
21344
21345// FastEmit functions for ISD::SUB.
21346
21347Register fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21348 if (RetVT.SimpleTy != MVT::i64)
21349 return Register();
21350 if ((Subtarget->hasNDD())) {
21351 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21352 }
21353 if ((!Subtarget->hasNDD())) {
21354 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21355 }
21356 return Register();
21357}
21358
21359Register fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21360 switch (VT.SimpleTy) {
21361 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21362 default: return Register();
21363 }
21364}
21365
21366// FastEmit functions for ISD::XOR.
21367
21368Register fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21369 if (RetVT.SimpleTy != MVT::i64)
21370 return Register();
21371 if ((Subtarget->hasNDD())) {
21372 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21373 }
21374 if ((!Subtarget->hasNDD())) {
21375 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21376 }
21377 return Register();
21378}
21379
21380Register fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21381 switch (VT.SimpleTy) {
21382 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21383 default: return Register();
21384 }
21385}
21386
21387// Top-level FastEmit function.
21388
21389Register fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
21390 switch (Opcode) {
21391 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21392 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21393 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21394 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21395 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21396 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21397 default: return Register();
21398 }
21399}
21400
21401// FastEmit functions for ISD::Constant.
21402
21403Register fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) {
21404 if (RetVT.SimpleTy != MVT::i8)
21405 return Register();
21406 return fastEmitInst_i(MachineInstOpcode: X86::MOV8ri, RC: &X86::GR8RegClass, Imm: imm0);
21407}
21408
21409Register fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) {
21410 if (RetVT.SimpleTy != MVT::i16)
21411 return Register();
21412 return fastEmitInst_i(MachineInstOpcode: X86::MOV16ri, RC: &X86::GR16RegClass, Imm: imm0);
21413}
21414
21415Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
21416 if (RetVT.SimpleTy != MVT::i32)
21417 return Register();
21418 return fastEmitInst_i(MachineInstOpcode: X86::MOV32ri, RC: &X86::GR32RegClass, Imm: imm0);
21419}
21420
21421Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
21422 if (RetVT.SimpleTy != MVT::i64)
21423 return Register();
21424 return fastEmitInst_i(MachineInstOpcode: X86::MOV64ri, RC: &X86::GR64RegClass, Imm: imm0);
21425}
21426
21427Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
21428 switch (VT.SimpleTy) {
21429 case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0);
21430 case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0);
21431 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
21432 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
21433 default: return Register();
21434 }
21435}
21436
21437// Top-level FastEmit function.
21438
21439Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
21440 switch (Opcode) {
21441 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
21442 default: return Register();
21443 }
21444}
21445
21446