1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the X86 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_AndMask64(int64_t Imm) {
12
13 return isMask_64(Value: Imm) && !isUInt<32>(x: Imm);
14
15}
16static bool Predicate_BTRMask64(int64_t Imm) {
17
18 return !isUInt<32>(x: Imm) && !isInt<32>(x: Imm) && isPowerOf2_64(Value: ~Imm);
19
20}
21static bool Predicate_BTCBTSMask64(int64_t Imm) {
22
23 return !isInt<32>(x: Imm) && isPowerOf2_64(Value: Imm);
24
25}
26static bool Predicate_i64immSExt32(int64_t Imm) {
27 return isInt<32>(x: Imm);
28}
29
30
31// FastEmit functions for ISD::ABS.
32
33Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
34 if (RetVT.SimpleTy != MVT::v16i8)
35 return Register();
36 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
37 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ128rr, RC: &X86::VR128XRegClass, Op0);
38 }
39 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
40 return fastEmitInst_r(MachineInstOpcode: X86::PABSBrr, RC: &X86::VR128RegClass, Op0);
41 }
42 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
43 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBrr, RC: &X86::VR128RegClass, Op0);
44 }
45 return Register();
46}
47
48Register fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, Register Op0) {
49 if (RetVT.SimpleTy != MVT::v32i8)
50 return Register();
51 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
52 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZ256rr, RC: &X86::VR256XRegClass, Op0);
53 }
54 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
55 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBYrr, RC: &X86::VR256RegClass, Op0);
56 }
57 return Register();
58}
59
60Register fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, Register Op0) {
61 if (RetVT.SimpleTy != MVT::v64i8)
62 return Register();
63 if ((Subtarget->hasBWI())) {
64 return fastEmitInst_r(MachineInstOpcode: X86::VPABSBZrr, RC: &X86::VR512RegClass, Op0);
65 }
66 return Register();
67}
68
69Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
70 if (RetVT.SimpleTy != MVT::v8i16)
71 return Register();
72 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
73 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ128rr, RC: &X86::VR128XRegClass, Op0);
74 }
75 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
76 return fastEmitInst_r(MachineInstOpcode: X86::PABSWrr, RC: &X86::VR128RegClass, Op0);
77 }
78 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
79 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWrr, RC: &X86::VR128RegClass, Op0);
80 }
81 return Register();
82}
83
84Register fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, Register Op0) {
85 if (RetVT.SimpleTy != MVT::v16i16)
86 return Register();
87 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
88 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZ256rr, RC: &X86::VR256XRegClass, Op0);
89 }
90 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
91 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWYrr, RC: &X86::VR256RegClass, Op0);
92 }
93 return Register();
94}
95
96Register fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, Register Op0) {
97 if (RetVT.SimpleTy != MVT::v32i16)
98 return Register();
99 if ((Subtarget->hasBWI())) {
100 return fastEmitInst_r(MachineInstOpcode: X86::VPABSWZrr, RC: &X86::VR512RegClass, Op0);
101 }
102 return Register();
103}
104
105Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
106 if (RetVT.SimpleTy != MVT::v4i32)
107 return Register();
108 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
109 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ128rr, RC: &X86::VR128XRegClass, Op0);
110 }
111 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
112 return fastEmitInst_r(MachineInstOpcode: X86::PABSDrr, RC: &X86::VR128RegClass, Op0);
113 }
114 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
115 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDrr, RC: &X86::VR128RegClass, Op0);
116 }
117 return Register();
118}
119
120Register fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, Register Op0) {
121 if (RetVT.SimpleTy != MVT::v8i32)
122 return Register();
123 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
124 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZ256rr, RC: &X86::VR256XRegClass, Op0);
125 }
126 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
127 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDYrr, RC: &X86::VR256RegClass, Op0);
128 }
129 return Register();
130}
131
132Register fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, Register Op0) {
133 if (RetVT.SimpleTy != MVT::v16i32)
134 return Register();
135 if ((Subtarget->hasAVX512())) {
136 return fastEmitInst_r(MachineInstOpcode: X86::VPABSDZrr, RC: &X86::VR512RegClass, Op0);
137 }
138 return Register();
139}
140
141Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
142 if (RetVT.SimpleTy != MVT::v2i64)
143 return Register();
144 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
145 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ128rr, RC: &X86::VR128XRegClass, Op0);
146 }
147 return Register();
148}
149
150Register fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, Register Op0) {
151 if (RetVT.SimpleTy != MVT::v4i64)
152 return Register();
153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
154 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZ256rr, RC: &X86::VR256XRegClass, Op0);
155 }
156 return Register();
157}
158
159Register fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, Register Op0) {
160 if (RetVT.SimpleTy != MVT::v8i64)
161 return Register();
162 if ((Subtarget->hasAVX512())) {
163 return fastEmitInst_r(MachineInstOpcode: X86::VPABSQZrr, RC: &X86::VR512RegClass, Op0);
164 }
165 return Register();
166}
167
168Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
169 switch (VT.SimpleTy) {
170 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
171 case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0);
172 case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0);
173 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
174 case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0);
175 case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0);
176 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
177 case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0);
178 case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0);
179 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
180 case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0);
181 case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0);
182 default: return Register();
183 }
184}
185
186// FastEmit functions for ISD::ANY_EXTEND.
187
188Register fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
189 if (RetVT.SimpleTy != MVT::i32)
190 return Register();
191 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
192}
193
194Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
195 switch (VT.SimpleTy) {
196 case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0);
197 default: return Register();
198 }
199}
200
201// FastEmit functions for ISD::BITCAST.
202
203Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
204 if (RetVT.SimpleTy != MVT::f32)
205 return Register();
206 if ((Subtarget->hasAVX512())) {
207 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSZrr, RC: &X86::FR32XRegClass, Op0);
208 }
209 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
210 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
211 }
212 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
213 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2SSrr, RC: &X86::FR32RegClass, Op0);
214 }
215 return Register();
216}
217
218Register fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Register Op0) {
219 if ((Subtarget->hasAVX512())) {
220 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDZrr, RC: &X86::FR64XRegClass, Op0);
221 }
222 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
223 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toSDrr, RC: &X86::FR64RegClass, Op0);
224 }
225 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
226 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toSDrr, RC: &X86::FR64RegClass, Op0);
227 }
228 return Register();
229}
230
231Register fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Register Op0) {
232 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
233 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64to64rr, RC: &X86::VR64RegClass, Op0);
234 }
235 return Register();
236}
237
238Register fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, Register Op0) {
239switch (RetVT.SimpleTy) {
240 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0);
241 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0);
242 default: return Register();
243}
244}
245
246Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
247 if (RetVT.SimpleTy != MVT::i32)
248 return Register();
249 if ((Subtarget->hasAVX512())) {
250 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIZrr, RC: &X86::GR32RegClass, Op0);
251 }
252 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
253 return fastEmitInst_r(MachineInstOpcode: X86::MOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
254 }
255 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
256 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSS2DIrr, RC: &X86::GR32RegClass, Op0);
257 }
258 return Register();
259}
260
261Register fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Register Op0) {
262 if ((Subtarget->hasAVX512())) {
263 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64Zrr, RC: &X86::GR64RegClass, Op0);
264 }
265 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
266 return fastEmitInst_r(MachineInstOpcode: X86::MOVSDto64rr, RC: &X86::GR64RegClass, Op0);
267 }
268 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
269 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSDto64rr, RC: &X86::GR64RegClass, Op0);
270 }
271 return Register();
272}
273
274Register fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Register Op0) {
275 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
276 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVFR642Qrr, RC: &X86::VR64RegClass, Op0);
277 }
278 return Register();
279}
280
281Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
282switch (RetVT.SimpleTy) {
283 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0);
284 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0);
285 default: return Register();
286}
287}
288
289Register fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Register Op0) {
290 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
291 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64from64rr, RC: &X86::GR64RegClass, Op0);
292 }
293 return Register();
294}
295
296Register fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Register Op0) {
297 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
298 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2FR64rr, RC: &X86::FR64RegClass, Op0);
299 }
300 return Register();
301}
302
303Register fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, Register Op0) {
304switch (RetVT.SimpleTy) {
305 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0);
306 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0);
307 default: return Register();
308}
309}
310
311Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
312 switch (VT.SimpleTy) {
313 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
314 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0);
315 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
316 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
317 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0);
318 default: return Register();
319 }
320}
321
322// FastEmit functions for ISD::BITREVERSE.
323
324Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
325 if (RetVT.SimpleTy != MVT::v16i8)
326 return Register();
327 if ((Subtarget->hasBMM()) && (Subtarget->hasVLX())) {
328 return fastEmitInst_r(MachineInstOpcode: X86::VPBITREVBZ128rr, RC: &X86::VR128XRegClass, Op0);
329 }
330 return Register();
331}
332
333Register fastEmit_ISD_BITREVERSE_MVT_v32i8_r(MVT RetVT, Register Op0) {
334 if (RetVT.SimpleTy != MVT::v32i8)
335 return Register();
336 if ((Subtarget->hasBMM()) && (Subtarget->hasVLX())) {
337 return fastEmitInst_r(MachineInstOpcode: X86::VPBITREVBZ256rr, RC: &X86::VR256XRegClass, Op0);
338 }
339 return Register();
340}
341
342Register fastEmit_ISD_BITREVERSE_MVT_v64i8_r(MVT RetVT, Register Op0) {
343 if (RetVT.SimpleTy != MVT::v64i8)
344 return Register();
345 if ((Subtarget->hasBMM())) {
346 return fastEmitInst_r(MachineInstOpcode: X86::VPBITREVBZrr, RC: &X86::VR512RegClass, Op0);
347 }
348 return Register();
349}
350
351Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
352 switch (VT.SimpleTy) {
353 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
354 case MVT::v32i8: return fastEmit_ISD_BITREVERSE_MVT_v32i8_r(RetVT, Op0);
355 case MVT::v64i8: return fastEmit_ISD_BITREVERSE_MVT_v64i8_r(RetVT, Op0);
356 default: return Register();
357 }
358}
359
360// FastEmit functions for ISD::BRIND.
361
362Register fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, Register Op0) {
363 if (RetVT.SimpleTy != MVT::isVoid)
364 return Register();
365 if ((!Subtarget->is64Bit())) {
366 return fastEmitInst_r(MachineInstOpcode: X86::JMP16r, RC: &X86::GR16RegClass, Op0);
367 }
368 return Register();
369}
370
371Register fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
372 if (RetVT.SimpleTy != MVT::isVoid)
373 return Register();
374 if ((!Subtarget->is64Bit())) {
375 return fastEmitInst_r(MachineInstOpcode: X86::JMP32r, RC: &X86::GR32RegClass, Op0);
376 }
377 return Register();
378}
379
380Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
381 if (RetVT.SimpleTy != MVT::isVoid)
382 return Register();
383 if ((Subtarget->is64Bit())) {
384 return fastEmitInst_r(MachineInstOpcode: X86::JMP64r, RC: &X86::GR64RegClass, Op0);
385 }
386 return Register();
387}
388
389Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
390 switch (VT.SimpleTy) {
391 case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0);
392 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
393 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
394 default: return Register();
395 }
396}
397
398// FastEmit functions for ISD::BSWAP.
399
400Register fastEmit_ISD_BSWAP_MVT_i16_r(MVT RetVT, Register Op0) {
401 if (RetVT.SimpleTy != MVT::i16)
402 return Register();
403 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
404 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE16rr, RC: &X86::GR16RegClass, Op0);
405 }
406 return Register();
407}
408
409Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
410 if (RetVT.SimpleTy != MVT::i32)
411 return Register();
412 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
413 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE32rr, RC: &X86::GR32RegClass, Op0);
414 }
415 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
416 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP32r, RC: &X86::GR32RegClass, Op0);
417 }
418 return Register();
419}
420
421Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
422 if (RetVT.SimpleTy != MVT::i64)
423 return Register();
424 if ((Subtarget->hasMOVBE()) && (Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
425 return fastEmitInst_r(MachineInstOpcode: X86::MOVBE64rr, RC: &X86::GR64RegClass, Op0);
426 }
427 if ((!Subtarget->hasNDD() || !Subtarget->hasMOVBE())) {
428 return fastEmitInst_r(MachineInstOpcode: X86::BSWAP64r, RC: &X86::GR64RegClass, Op0);
429 }
430 return Register();
431}
432
433Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
434 switch (VT.SimpleTy) {
435 case MVT::i16: return fastEmit_ISD_BSWAP_MVT_i16_r(RetVT, Op0);
436 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
437 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
438 default: return Register();
439 }
440}
441
442// FastEmit functions for ISD::CTLZ.
443
444Register fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, Register Op0) {
445 if (RetVT.SimpleTy != MVT::i16)
446 return Register();
447 if ((Subtarget->hasLZCNT())) {
448 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT16rr, RC: &X86::GR16RegClass, Op0);
449 }
450 return Register();
451}
452
453Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
454 if (RetVT.SimpleTy != MVT::i32)
455 return Register();
456 if ((Subtarget->hasLZCNT())) {
457 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT32rr, RC: &X86::GR32RegClass, Op0);
458 }
459 return Register();
460}
461
462Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
463 if (RetVT.SimpleTy != MVT::i64)
464 return Register();
465 if ((Subtarget->hasLZCNT())) {
466 return fastEmitInst_r(MachineInstOpcode: X86::LZCNT64rr, RC: &X86::GR64RegClass, Op0);
467 }
468 return Register();
469}
470
471Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
472 if (RetVT.SimpleTy != MVT::v4i32)
473 return Register();
474 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
475 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
476 }
477 return Register();
478}
479
480Register fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, Register Op0) {
481 if (RetVT.SimpleTy != MVT::v8i32)
482 return Register();
483 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
484 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
485 }
486 return Register();
487}
488
489Register fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, Register Op0) {
490 if (RetVT.SimpleTy != MVT::v16i32)
491 return Register();
492 if ((Subtarget->hasCDI())) {
493 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTDZrr, RC: &X86::VR512RegClass, Op0);
494 }
495 return Register();
496}
497
498Register fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, Register Op0) {
499 if (RetVT.SimpleTy != MVT::v2i64)
500 return Register();
501 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
502 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
503 }
504 return Register();
505}
506
507Register fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, Register Op0) {
508 if (RetVT.SimpleTy != MVT::v4i64)
509 return Register();
510 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
511 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
512 }
513 return Register();
514}
515
516Register fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, Register Op0) {
517 if (RetVT.SimpleTy != MVT::v8i64)
518 return Register();
519 if ((Subtarget->hasCDI())) {
520 return fastEmitInst_r(MachineInstOpcode: X86::VPLZCNTQZrr, RC: &X86::VR512RegClass, Op0);
521 }
522 return Register();
523}
524
525Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
526 switch (VT.SimpleTy) {
527 case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0);
528 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
529 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
530 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
531 case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0);
532 case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0);
533 case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0);
534 case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0);
535 case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0);
536 default: return Register();
537 }
538}
539
540// FastEmit functions for ISD::CTPOP.
541
542Register fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, Register Op0) {
543 if (RetVT.SimpleTy != MVT::i16)
544 return Register();
545 if ((Subtarget->hasPOPCNT())) {
546 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT16rr, RC: &X86::GR16RegClass, Op0);
547 }
548 return Register();
549}
550
551Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
552 if (RetVT.SimpleTy != MVT::i32)
553 return Register();
554 if ((Subtarget->hasPOPCNT())) {
555 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT32rr, RC: &X86::GR32RegClass, Op0);
556 }
557 return Register();
558}
559
560Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
561 if (RetVT.SimpleTy != MVT::i64)
562 return Register();
563 if ((Subtarget->hasPOPCNT())) {
564 return fastEmitInst_r(MachineInstOpcode: X86::POPCNT64rr, RC: &X86::GR64RegClass, Op0);
565 }
566 return Register();
567}
568
569Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
570 if (RetVT.SimpleTy != MVT::v16i8)
571 return Register();
572 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
573 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ128rr, RC: &X86::VR128XRegClass, Op0);
574 }
575 return Register();
576}
577
578Register fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, Register Op0) {
579 if (RetVT.SimpleTy != MVT::v32i8)
580 return Register();
581 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
582 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZ256rr, RC: &X86::VR256XRegClass, Op0);
583 }
584 return Register();
585}
586
587Register fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, Register Op0) {
588 if (RetVT.SimpleTy != MVT::v64i8)
589 return Register();
590 if ((Subtarget->hasBITALG())) {
591 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTBZrr, RC: &X86::VR512RegClass, Op0);
592 }
593 return Register();
594}
595
596Register fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, Register Op0) {
597 if (RetVT.SimpleTy != MVT::v8i16)
598 return Register();
599 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
600 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ128rr, RC: &X86::VR128XRegClass, Op0);
601 }
602 return Register();
603}
604
605Register fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, Register Op0) {
606 if (RetVT.SimpleTy != MVT::v16i16)
607 return Register();
608 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
609 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZ256rr, RC: &X86::VR256XRegClass, Op0);
610 }
611 return Register();
612}
613
614Register fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, Register Op0) {
615 if (RetVT.SimpleTy != MVT::v32i16)
616 return Register();
617 if ((Subtarget->hasBITALG())) {
618 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTWZrr, RC: &X86::VR512RegClass, Op0);
619 }
620 return Register();
621}
622
623Register fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, Register Op0) {
624 if (RetVT.SimpleTy != MVT::v4i32)
625 return Register();
626 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
627 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ128rr, RC: &X86::VR128XRegClass, Op0);
628 }
629 return Register();
630}
631
632Register fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, Register Op0) {
633 if (RetVT.SimpleTy != MVT::v8i32)
634 return Register();
635 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
636 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZ256rr, RC: &X86::VR256XRegClass, Op0);
637 }
638 return Register();
639}
640
641Register fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, Register Op0) {
642 if (RetVT.SimpleTy != MVT::v16i32)
643 return Register();
644 if ((Subtarget->hasVPOPCNTDQ())) {
645 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTDZrr, RC: &X86::VR512RegClass, Op0);
646 }
647 return Register();
648}
649
650Register fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, Register Op0) {
651 if (RetVT.SimpleTy != MVT::v2i64)
652 return Register();
653 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
654 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ128rr, RC: &X86::VR128XRegClass, Op0);
655 }
656 return Register();
657}
658
659Register fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, Register Op0) {
660 if (RetVT.SimpleTy != MVT::v4i64)
661 return Register();
662 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
663 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZ256rr, RC: &X86::VR256XRegClass, Op0);
664 }
665 return Register();
666}
667
668Register fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, Register Op0) {
669 if (RetVT.SimpleTy != MVT::v8i64)
670 return Register();
671 if ((Subtarget->hasVPOPCNTDQ())) {
672 return fastEmitInst_r(MachineInstOpcode: X86::VPOPCNTQZrr, RC: &X86::VR512RegClass, Op0);
673 }
674 return Register();
675}
676
677Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
678 switch (VT.SimpleTy) {
679 case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0);
680 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
681 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
682 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
683 case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0);
684 case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0);
685 case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0);
686 case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0);
687 case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0);
688 case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0);
689 case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0);
690 case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0);
691 case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0);
692 case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0);
693 case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0);
694 default: return Register();
695 }
696}
697
698// FastEmit functions for ISD::CTTZ.
699
700Register fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, Register Op0) {
701 if (RetVT.SimpleTy != MVT::i16)
702 return Register();
703 if ((Subtarget->hasBMI())) {
704 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT16rr, RC: &X86::GR16RegClass, Op0);
705 }
706 return Register();
707}
708
709Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
710 if (RetVT.SimpleTy != MVT::i32)
711 return Register();
712 if ((Subtarget->hasBMI())) {
713 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT32rr, RC: &X86::GR32RegClass, Op0);
714 }
715 return Register();
716}
717
718Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
719 if (RetVT.SimpleTy != MVT::i64)
720 return Register();
721 if ((Subtarget->hasBMI())) {
722 return fastEmitInst_r(MachineInstOpcode: X86::TZCNT64rr, RC: &X86::GR64RegClass, Op0);
723 }
724 return Register();
725}
726
727Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
728 switch (VT.SimpleTy) {
729 case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0);
730 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
731 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
732 default: return Register();
733 }
734}
735
736// FastEmit functions for ISD::FABS.
737
738Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
739 if (RetVT.SimpleTy != MVT::f32)
740 return Register();
741 if ((!Subtarget->hasSSE1())) {
742 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp32, RC: &X86::RFP32RegClass, Op0);
743 }
744 return Register();
745}
746
747Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
748 if (RetVT.SimpleTy != MVT::f64)
749 return Register();
750 if ((!Subtarget->hasSSE2())) {
751 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp64, RC: &X86::RFP64RegClass, Op0);
752 }
753 return Register();
754}
755
756Register fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, Register Op0) {
757 if (RetVT.SimpleTy != MVT::f80)
758 return Register();
759 if ((Subtarget->hasX87())) {
760 return fastEmitInst_r(MachineInstOpcode: X86::ABS_Fp80, RC: &X86::RFP80RegClass, Op0);
761 }
762 return Register();
763}
764
765Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
766 switch (VT.SimpleTy) {
767 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
768 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
769 case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0);
770 default: return Register();
771 }
772}
773
774// FastEmit functions for ISD::FNEG.
775
776Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
777 if (RetVT.SimpleTy != MVT::f32)
778 return Register();
779 if ((!Subtarget->hasSSE1())) {
780 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp32, RC: &X86::RFP32RegClass, Op0);
781 }
782 return Register();
783}
784
785Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
786 if (RetVT.SimpleTy != MVT::f64)
787 return Register();
788 if ((!Subtarget->hasSSE2())) {
789 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp64, RC: &X86::RFP64RegClass, Op0);
790 }
791 return Register();
792}
793
794Register fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, Register Op0) {
795 if (RetVT.SimpleTy != MVT::f80)
796 return Register();
797 if ((Subtarget->hasX87())) {
798 return fastEmitInst_r(MachineInstOpcode: X86::CHS_Fp80, RC: &X86::RFP80RegClass, Op0);
799 }
800 return Register();
801}
802
803Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
804 switch (VT.SimpleTy) {
805 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
806 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
807 case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0);
808 default: return Register();
809 }
810}
811
812// FastEmit functions for ISD::FP_EXTEND.
813
814Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
815 if (RetVT.SimpleTy != MVT::f64)
816 return Register();
817 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
818 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
819 }
820 return Register();
821}
822
823Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Register Op0) {
824 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
825 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
826 }
827 return Register();
828}
829
830Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Register Op0) {
831 if ((Subtarget->hasFP16())) {
832 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
833 }
834 return Register();
835}
836
837Register fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(MVT RetVT, Register Op0) {
838switch (RetVT.SimpleTy) {
839 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
840 case MVT::v8f64: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
841 default: return Register();
842}
843}
844
845Register fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(MVT RetVT, Register Op0) {
846 if (RetVT.SimpleTy != MVT::v16f32)
847 return Register();
848 if ((Subtarget->hasFP16())) {
849 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
850 }
851 return Register();
852}
853
854Register fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, Register Op0) {
855 if (RetVT.SimpleTy != MVT::v4f64)
856 return Register();
857 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
858 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
859 }
860 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
861 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
862 }
863 return Register();
864}
865
866Register fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, Register Op0) {
867 if (RetVT.SimpleTy != MVT::v8f64)
868 return Register();
869 if ((Subtarget->hasAVX512())) {
870 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
871 }
872 return Register();
873}
874
875Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
876 switch (VT.SimpleTy) {
877 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
878 case MVT::v8f16: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
879 case MVT::v16f16: return fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
880 case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
881 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
882 default: return Register();
883 }
884}
885
886// FastEmit functions for ISD::FP_ROUND.
887
888Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
889 if (RetVT.SimpleTy != MVT::f32)
890 return Register();
891 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
892 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
893 }
894 return Register();
895}
896
897Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
898 switch (VT.SimpleTy) {
899 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
900 default: return Register();
901 }
902}
903
904// FastEmit functions for ISD::FP_TO_SINT.
905
906Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
907 if ((Subtarget->hasFP16())) {
908 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
909 }
910 return Register();
911}
912
913Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
914 if ((Subtarget->hasFP16())) {
915 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
916 }
917 return Register();
918}
919
920Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
921switch (RetVT.SimpleTy) {
922 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
923 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
924 default: return Register();
925}
926}
927
928Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
929 if ((Subtarget->hasAVX512())) {
930 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
931 }
932 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
933 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
934 }
935 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
936 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
937 }
938 return Register();
939}
940
941Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
942 if ((Subtarget->hasAVX512())) {
943 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
944 }
945 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
946 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
947 }
948 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
949 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
950 }
951 return Register();
952}
953
954Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
955switch (RetVT.SimpleTy) {
956 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
957 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
958 default: return Register();
959}
960}
961
962Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
963 if ((Subtarget->hasAVX512())) {
964 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
965 }
966 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
967 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
968 }
969 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
970 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
971 }
972 return Register();
973}
974
975Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
976 if ((Subtarget->hasAVX512())) {
977 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
978 }
979 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
980 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
981 }
982 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
983 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
984 }
985 return Register();
986}
987
988Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
989switch (RetVT.SimpleTy) {
990 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
991 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
992 default: return Register();
993}
994}
995
996Register fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
997 if (RetVT.SimpleTy != MVT::v4i32)
998 return Register();
999 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1000 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
1001 }
1002 return Register();
1003}
1004
1005Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
1006 switch (VT.SimpleTy) {
1007 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
1008 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
1009 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
1010 case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
1011 default: return Register();
1012 }
1013}
1014
1015// FastEmit functions for ISD::FP_TO_UINT.
1016
1017Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
1018 if ((Subtarget->hasFP16())) {
1019 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
1020 }
1021 return Register();
1022}
1023
1024Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
1025 if ((Subtarget->hasFP16())) {
1026 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1027 }
1028 return Register();
1029}
1030
1031Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
1032switch (RetVT.SimpleTy) {
1033 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
1034 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
1035 default: return Register();
1036}
1037}
1038
1039Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
1040 if ((Subtarget->hasAVX512())) {
1041 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
1042 }
1043 return Register();
1044}
1045
1046Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
1047 if ((Subtarget->hasAVX512())) {
1048 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1049 }
1050 return Register();
1051}
1052
1053Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
1054switch (RetVT.SimpleTy) {
1055 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
1056 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
1057 default: return Register();
1058}
1059}
1060
1061Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
1062 if ((Subtarget->hasAVX512())) {
1063 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
1064 }
1065 return Register();
1066}
1067
1068Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
1069 if ((Subtarget->hasAVX512())) {
1070 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
1071 }
1072 return Register();
1073}
1074
1075Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
1076switch (RetVT.SimpleTy) {
1077 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
1078 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
1079 default: return Register();
1080}
1081}
1082
1083Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
1084 switch (VT.SimpleTy) {
1085 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
1086 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
1087 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
1088 default: return Register();
1089 }
1090}
1091
1092// FastEmit functions for ISD::FSQRT.
1093
1094Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
1095 if (RetVT.SimpleTy != MVT::f32)
1096 return Register();
1097 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1098 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
1099 }
1100 if ((!Subtarget->hasSSE1())) {
1101 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
1102 }
1103 return Register();
1104}
1105
1106Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
1107 if (RetVT.SimpleTy != MVT::f64)
1108 return Register();
1109 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1110 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
1111 }
1112 if ((!Subtarget->hasSSE2())) {
1113 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
1114 }
1115 return Register();
1116}
1117
1118Register fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, Register Op0) {
1119 if (RetVT.SimpleTy != MVT::f80)
1120 return Register();
1121 if ((Subtarget->hasX87())) {
1122 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
1123 }
1124 return Register();
1125}
1126
1127Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1128 if (RetVT.SimpleTy != MVT::v8f16)
1129 return Register();
1130 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1131 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
1132 }
1133 return Register();
1134}
1135
1136Register fastEmit_ISD_FSQRT_MVT_v16f16_r(MVT RetVT, Register Op0) {
1137 if (RetVT.SimpleTy != MVT::v16f16)
1138 return Register();
1139 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1140 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
1141 }
1142 return Register();
1143}
1144
1145Register fastEmit_ISD_FSQRT_MVT_v32f16_r(MVT RetVT, Register Op0) {
1146 if (RetVT.SimpleTy != MVT::v32f16)
1147 return Register();
1148 if ((Subtarget->hasFP16())) {
1149 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
1150 }
1151 return Register();
1152}
1153
1154Register fastEmit_ISD_FSQRT_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1155 if (RetVT.SimpleTy != MVT::v8bf16)
1156 return Register();
1157 if ((Subtarget->hasAVX10_2())) {
1158 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
1159 }
1160 return Register();
1161}
1162
1163Register fastEmit_ISD_FSQRT_MVT_v16bf16_r(MVT RetVT, Register Op0) {
1164 if (RetVT.SimpleTy != MVT::v16bf16)
1165 return Register();
1166 if ((Subtarget->hasAVX10_2())) {
1167 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
1168 }
1169 return Register();
1170}
1171
1172Register fastEmit_ISD_FSQRT_MVT_v32bf16_r(MVT RetVT, Register Op0) {
1173 if (RetVT.SimpleTy != MVT::v32bf16)
1174 return Register();
1175 if ((Subtarget->hasAVX10_2())) {
1176 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
1177 }
1178 return Register();
1179}
1180
1181Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1182 if (RetVT.SimpleTy != MVT::v4f32)
1183 return Register();
1184 if ((Subtarget->hasVLX())) {
1185 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
1186 }
1187 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1188 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
1189 }
1190 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1191 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
1192 }
1193 return Register();
1194}
1195
1196Register fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1197 if (RetVT.SimpleTy != MVT::v8f32)
1198 return Register();
1199 if ((Subtarget->hasVLX())) {
1200 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
1201 }
1202 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1203 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
1204 }
1205 return Register();
1206}
1207
1208Register fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, Register Op0) {
1209 if (RetVT.SimpleTy != MVT::v16f32)
1210 return Register();
1211 if ((Subtarget->hasAVX512())) {
1212 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
1213 }
1214 return Register();
1215}
1216
1217Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1218 if (RetVT.SimpleTy != MVT::v2f64)
1219 return Register();
1220 if ((Subtarget->hasVLX())) {
1221 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
1222 }
1223 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1224 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
1225 }
1226 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1227 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
1228 }
1229 return Register();
1230}
1231
1232Register fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1233 if (RetVT.SimpleTy != MVT::v4f64)
1234 return Register();
1235 if ((Subtarget->hasVLX())) {
1236 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
1237 }
1238 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1239 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
1240 }
1241 return Register();
1242}
1243
1244Register fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1245 if (RetVT.SimpleTy != MVT::v8f64)
1246 return Register();
1247 if ((Subtarget->hasAVX512())) {
1248 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
1249 }
1250 return Register();
1251}
1252
1253Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
1254 switch (VT.SimpleTy) {
1255 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
1256 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
1257 case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0);
1258 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
1259 case MVT::v16f16: return fastEmit_ISD_FSQRT_MVT_v16f16_r(RetVT, Op0);
1260 case MVT::v32f16: return fastEmit_ISD_FSQRT_MVT_v32f16_r(RetVT, Op0);
1261 case MVT::v8bf16: return fastEmit_ISD_FSQRT_MVT_v8bf16_r(RetVT, Op0);
1262 case MVT::v16bf16: return fastEmit_ISD_FSQRT_MVT_v16bf16_r(RetVT, Op0);
1263 case MVT::v32bf16: return fastEmit_ISD_FSQRT_MVT_v32bf16_r(RetVT, Op0);
1264 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
1265 case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0);
1266 case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0);
1267 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
1268 case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0);
1269 case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0);
1270 default: return Register();
1271 }
1272}
1273
1274// FastEmit functions for ISD::LLRINT.
1275
1276Register fastEmit_ISD_LLRINT_MVT_f16_r(MVT RetVT, Register Op0) {
1277 if (RetVT.SimpleTy != MVT::i64)
1278 return Register();
1279 if ((Subtarget->hasFP16()) && (Subtarget->is64Bit())) {
1280 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1281 }
1282 return Register();
1283}
1284
1285Register fastEmit_ISD_LLRINT_MVT_f32_r(MVT RetVT, Register Op0) {
1286 if (RetVT.SimpleTy != MVT::i64)
1287 return Register();
1288 if ((Subtarget->hasAVX512())) {
1289 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1290 }
1291 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1292 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1293 }
1294 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1295 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1296 }
1297 return Register();
1298}
1299
1300Register fastEmit_ISD_LLRINT_MVT_f64_r(MVT RetVT, Register Op0) {
1301 if (RetVT.SimpleTy != MVT::i64)
1302 return Register();
1303 if ((Subtarget->hasAVX512())) {
1304 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1305 }
1306 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1307 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1308 }
1309 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1310 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1311 }
1312 return Register();
1313}
1314
1315Register fastEmit_ISD_LLRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1316 if (RetVT.SimpleTy != MVT::v8i64)
1317 return Register();
1318 if ((Subtarget->hasFP16())) {
1319 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
1320 }
1321 return Register();
1322}
1323
1324Register fastEmit_ISD_LLRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1325 if (RetVT.SimpleTy != MVT::v4i64)
1326 return Register();
1327 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1328 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1329 }
1330 return Register();
1331}
1332
1333Register fastEmit_ISD_LLRINT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1334 if (RetVT.SimpleTy != MVT::v8i64)
1335 return Register();
1336 if ((Subtarget->hasDQI())) {
1337 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1338 }
1339 return Register();
1340}
1341
1342Register fastEmit_ISD_LLRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1343 if (RetVT.SimpleTy != MVT::v2i64)
1344 return Register();
1345 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1346 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1347 }
1348 return Register();
1349}
1350
1351Register fastEmit_ISD_LLRINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1352 if (RetVT.SimpleTy != MVT::v4i64)
1353 return Register();
1354 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1355 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1356 }
1357 return Register();
1358}
1359
1360Register fastEmit_ISD_LLRINT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1361 if (RetVT.SimpleTy != MVT::v8i64)
1362 return Register();
1363 if ((Subtarget->hasDQI())) {
1364 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1365 }
1366 return Register();
1367}
1368
1369Register fastEmit_ISD_LLRINT_r(MVT VT, MVT RetVT, Register Op0) {
1370 switch (VT.SimpleTy) {
1371 case MVT::f16: return fastEmit_ISD_LLRINT_MVT_f16_r(RetVT, Op0);
1372 case MVT::f32: return fastEmit_ISD_LLRINT_MVT_f32_r(RetVT, Op0);
1373 case MVT::f64: return fastEmit_ISD_LLRINT_MVT_f64_r(RetVT, Op0);
1374 case MVT::v8f16: return fastEmit_ISD_LLRINT_MVT_v8f16_r(RetVT, Op0);
1375 case MVT::v4f32: return fastEmit_ISD_LLRINT_MVT_v4f32_r(RetVT, Op0);
1376 case MVT::v8f32: return fastEmit_ISD_LLRINT_MVT_v8f32_r(RetVT, Op0);
1377 case MVT::v2f64: return fastEmit_ISD_LLRINT_MVT_v2f64_r(RetVT, Op0);
1378 case MVT::v4f64: return fastEmit_ISD_LLRINT_MVT_v4f64_r(RetVT, Op0);
1379 case MVT::v8f64: return fastEmit_ISD_LLRINT_MVT_v8f64_r(RetVT, Op0);
1380 default: return Register();
1381 }
1382}
1383
1384// FastEmit functions for ISD::LRINT.
1385
1386Register fastEmit_ISD_LRINT_MVT_f16_MVT_i32_r(Register Op0) {
1387 if ((Subtarget->hasFP16())) {
1388 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
1389 }
1390 return Register();
1391}
1392
1393Register fastEmit_ISD_LRINT_MVT_f16_MVT_i64_r(Register Op0) {
1394 if ((Subtarget->hasFP16()) && (Subtarget->is64Bit())) {
1395 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1396 }
1397 return Register();
1398}
1399
1400Register fastEmit_ISD_LRINT_MVT_f16_r(MVT RetVT, Register Op0) {
1401switch (RetVT.SimpleTy) {
1402 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f16_MVT_i32_r(Op0);
1403 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f16_MVT_i64_r(Op0);
1404 default: return Register();
1405}
1406}
1407
1408Register fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Register Op0) {
1409 if ((Subtarget->hasAVX512())) {
1410 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
1411 }
1412 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1413 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1414 }
1415 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1416 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr, RC: &X86::GR32RegClass, Op0);
1417 }
1418 return Register();
1419}
1420
1421Register fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Register Op0) {
1422 if ((Subtarget->hasAVX512())) {
1423 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1424 }
1425 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1426 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1427 }
1428 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1429 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
1430 }
1431 return Register();
1432}
1433
1434Register fastEmit_ISD_LRINT_MVT_f32_r(MVT RetVT, Register Op0) {
1435switch (RetVT.SimpleTy) {
1436 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Op0);
1437 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Op0);
1438 default: return Register();
1439}
1440}
1441
1442Register fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Register Op0) {
1443 if ((Subtarget->hasAVX512())) {
1444 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
1445 }
1446 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1447 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1448 }
1449 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1450 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr, RC: &X86::GR32RegClass, Op0);
1451 }
1452 return Register();
1453}
1454
1455Register fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Register Op0) {
1456 if ((Subtarget->hasAVX512())) {
1457 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
1458 }
1459 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1460 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1461 }
1462 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1463 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
1464 }
1465 return Register();
1466}
1467
1468Register fastEmit_ISD_LRINT_MVT_f64_r(MVT RetVT, Register Op0) {
1469switch (RetVT.SimpleTy) {
1470 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Op0);
1471 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Op0);
1472 default: return Register();
1473}
1474}
1475
1476Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i16_r(Register Op0) {
1477 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1478 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
1479 }
1480 return Register();
1481}
1482
1483Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i32_r(Register Op0) {
1484 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1485 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1486 }
1487 return Register();
1488}
1489
1490Register fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i64_r(Register Op0) {
1491 if ((Subtarget->hasFP16())) {
1492 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
1493 }
1494 return Register();
1495}
1496
1497Register fastEmit_ISD_LRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
1498switch (RetVT.SimpleTy) {
1499 case MVT::v8i16: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i16_r(Op0);
1500 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i32_r(Op0);
1501 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f16_MVT_v8i64_r(Op0);
1502 default: return Register();
1503}
1504}
1505
1506Register fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i16_r(Register Op0) {
1507 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1508 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
1509 }
1510 return Register();
1511}
1512
1513Register fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i32_r(Register Op0) {
1514 if ((Subtarget->hasFP16())) {
1515 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
1516 }
1517 return Register();
1518}
1519
1520Register fastEmit_ISD_LRINT_MVT_v16f16_r(MVT RetVT, Register Op0) {
1521switch (RetVT.SimpleTy) {
1522 case MVT::v16i16: return fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i16_r(Op0);
1523 case MVT::v16i32: return fastEmit_ISD_LRINT_MVT_v16f16_MVT_v16i32_r(Op0);
1524 default: return Register();
1525}
1526}
1527
1528Register fastEmit_ISD_LRINT_MVT_v32f16_r(MVT RetVT, Register Op0) {
1529 if (RetVT.SimpleTy != MVT::v32i16)
1530 return Register();
1531 if ((Subtarget->hasFP16())) {
1532 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
1533 }
1534 return Register();
1535}
1536
1537Register fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Register Op0) {
1538 if ((Subtarget->hasVLX())) {
1539 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
1540 }
1541 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1542 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1543 }
1544 if ((Subtarget->hasAVX())) {
1545 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
1546 }
1547 return Register();
1548}
1549
1550Register fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Register Op0) {
1551 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1552 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1553 }
1554 return Register();
1555}
1556
1557Register fastEmit_ISD_LRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
1558switch (RetVT.SimpleTy) {
1559 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i32_r(Op0);
1560 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f32_MVT_v4i64_r(Op0);
1561 default: return Register();
1562}
1563}
1564
1565Register fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Register Op0) {
1566 if ((Subtarget->hasVLX())) {
1567 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
1568 }
1569 if ((Subtarget->hasAVX())) {
1570 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
1571 }
1572 return Register();
1573}
1574
1575Register fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Register Op0) {
1576 if ((Subtarget->hasDQI())) {
1577 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
1578 }
1579 return Register();
1580}
1581
1582Register fastEmit_ISD_LRINT_MVT_v8f32_r(MVT RetVT, Register Op0) {
1583switch (RetVT.SimpleTy) {
1584 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i32_r(Op0);
1585 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f32_MVT_v8i64_r(Op0);
1586 default: return Register();
1587}
1588}
1589
1590Register fastEmit_ISD_LRINT_MVT_v16f32_r(MVT RetVT, Register Op0) {
1591 if (RetVT.SimpleTy != MVT::v16i32)
1592 return Register();
1593 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
1594}
1595
1596Register fastEmit_ISD_LRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
1597 if (RetVT.SimpleTy != MVT::v2i64)
1598 return Register();
1599 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1600 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
1601 }
1602 return Register();
1603}
1604
1605Register fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Register Op0) {
1606 if ((Subtarget->hasVLX())) {
1607 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
1608 }
1609 if ((Subtarget->hasAVX())) {
1610 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
1611 }
1612 return Register();
1613}
1614
1615Register fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Register Op0) {
1616 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1617 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
1618 }
1619 return Register();
1620}
1621
1622Register fastEmit_ISD_LRINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
1623switch (RetVT.SimpleTy) {
1624 case MVT::v4i32: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i32_r(Op0);
1625 case MVT::v4i64: return fastEmit_ISD_LRINT_MVT_v4f64_MVT_v4i64_r(Op0);
1626 default: return Register();
1627}
1628}
1629
1630Register fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Register Op0) {
1631 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
1632}
1633
1634Register fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Register Op0) {
1635 if ((Subtarget->hasDQI())) {
1636 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
1637 }
1638 return Register();
1639}
1640
1641Register fastEmit_ISD_LRINT_MVT_v8f64_r(MVT RetVT, Register Op0) {
1642switch (RetVT.SimpleTy) {
1643 case MVT::v8i32: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i32_r(Op0);
1644 case MVT::v8i64: return fastEmit_ISD_LRINT_MVT_v8f64_MVT_v8i64_r(Op0);
1645 default: return Register();
1646}
1647}
1648
1649Register fastEmit_ISD_LRINT_r(MVT VT, MVT RetVT, Register Op0) {
1650 switch (VT.SimpleTy) {
1651 case MVT::f16: return fastEmit_ISD_LRINT_MVT_f16_r(RetVT, Op0);
1652 case MVT::f32: return fastEmit_ISD_LRINT_MVT_f32_r(RetVT, Op0);
1653 case MVT::f64: return fastEmit_ISD_LRINT_MVT_f64_r(RetVT, Op0);
1654 case MVT::v8f16: return fastEmit_ISD_LRINT_MVT_v8f16_r(RetVT, Op0);
1655 case MVT::v16f16: return fastEmit_ISD_LRINT_MVT_v16f16_r(RetVT, Op0);
1656 case MVT::v32f16: return fastEmit_ISD_LRINT_MVT_v32f16_r(RetVT, Op0);
1657 case MVT::v4f32: return fastEmit_ISD_LRINT_MVT_v4f32_r(RetVT, Op0);
1658 case MVT::v8f32: return fastEmit_ISD_LRINT_MVT_v8f32_r(RetVT, Op0);
1659 case MVT::v16f32: return fastEmit_ISD_LRINT_MVT_v16f32_r(RetVT, Op0);
1660 case MVT::v2f64: return fastEmit_ISD_LRINT_MVT_v2f64_r(RetVT, Op0);
1661 case MVT::v4f64: return fastEmit_ISD_LRINT_MVT_v4f64_r(RetVT, Op0);
1662 case MVT::v8f64: return fastEmit_ISD_LRINT_MVT_v8f64_r(RetVT, Op0);
1663 default: return Register();
1664 }
1665}
1666
1667// FastEmit functions for ISD::SCALAR_TO_VECTOR.
1668
1669Register fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
1670 if (RetVT.SimpleTy != MVT::v4i32)
1671 return Register();
1672 if ((Subtarget->hasAVX512())) {
1673 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIZrr, RC: &X86::VR128XRegClass, Op0);
1674 }
1675 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1676 return fastEmitInst_r(MachineInstOpcode: X86::MOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1677 }
1678 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1679 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDI2PDIrr, RC: &X86::VR128RegClass, Op0);
1680 }
1681 return Register();
1682}
1683
1684Register fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
1685 if (RetVT.SimpleTy != MVT::v2i64)
1686 return Register();
1687 if ((Subtarget->hasAVX512())) {
1688 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIZrr, RC: &X86::VR128XRegClass, Op0);
1689 }
1690 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1691 return fastEmitInst_r(MachineInstOpcode: X86::MOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1692 }
1693 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1694 return fastEmitInst_r(MachineInstOpcode: X86::VMOV64toPQIrr, RC: &X86::VR128RegClass, Op0);
1695 }
1696 return Register();
1697}
1698
1699Register fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
1700 switch (VT.SimpleTy) {
1701 case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0);
1702 case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0);
1703 default: return Register();
1704 }
1705}
1706
1707// FastEmit functions for ISD::SIGN_EXTEND.
1708
1709Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Register Op0) {
1710 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr8, RC: &X86::GR32RegClass, Op0);
1711}
1712
1713Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Register Op0) {
1714 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr8, RC: &X86::GR64RegClass, Op0);
1715}
1716
1717Register fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
1718switch (RetVT.SimpleTy) {
1719 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0);
1720 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0);
1721 default: return Register();
1722}
1723}
1724
1725Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Register Op0) {
1726 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX32rr16, RC: &X86::GR32RegClass, Op0);
1727}
1728
1729Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Register Op0) {
1730 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr16, RC: &X86::GR64RegClass, Op0);
1731}
1732
1733Register fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, Register Op0) {
1734switch (RetVT.SimpleTy) {
1735 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0);
1736 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0);
1737 default: return Register();
1738}
1739}
1740
1741Register fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, Register Op0) {
1742 if (RetVT.SimpleTy != MVT::i64)
1743 return Register();
1744 if ((Subtarget->is64Bit())) {
1745 return fastEmitInst_r(MachineInstOpcode: X86::MOVSX64rr32, RC: &X86::GR64RegClass, Op0);
1746 }
1747 return Register();
1748}
1749
1750Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, Register Op0) {
1751 if (RetVT.SimpleTy != MVT::v2i64)
1752 return Register();
1753 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1754 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ128rk, RC: &X86::VR128XRegClass, Op0);
1755 }
1756 return Register();
1757}
1758
1759Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Register Op0) {
1760 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1761 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ128rk, RC: &X86::VR128XRegClass, Op0);
1762 }
1763 return Register();
1764}
1765
1766Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Register Op0) {
1767 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1768 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZ256rk, RC: &X86::VR256XRegClass, Op0);
1769 }
1770 return Register();
1771}
1772
1773Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, Register Op0) {
1774switch (RetVT.SimpleTy) {
1775 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0);
1776 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0);
1777 default: return Register();
1778}
1779}
1780
1781Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Register Op0) {
1782 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1783 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ128rk, RC: &X86::VR128XRegClass, Op0);
1784 }
1785 return Register();
1786}
1787
1788Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Register Op0) {
1789 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1790 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZ256rk, RC: &X86::VR256XRegClass, Op0);
1791 }
1792 return Register();
1793}
1794
1795Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Register Op0) {
1796 if ((Subtarget->hasDQI())) {
1797 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2QZrk, RC: &X86::VR512RegClass, Op0);
1798 }
1799 return Register();
1800}
1801
1802Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, Register Op0) {
1803switch (RetVT.SimpleTy) {
1804 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0);
1805 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0);
1806 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0);
1807 default: return Register();
1808}
1809}
1810
1811Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Register Op0) {
1812 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1813 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ128rk, RC: &X86::VR128XRegClass, Op0);
1814 }
1815 return Register();
1816}
1817
1818Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Register Op0) {
1819 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1820 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZ256rk, RC: &X86::VR256XRegClass, Op0);
1821 }
1822 return Register();
1823}
1824
1825Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Register Op0) {
1826 if ((Subtarget->hasDQI())) {
1827 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2DZrk, RC: &X86::VR512RegClass, Op0);
1828 }
1829 return Register();
1830}
1831
1832Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, Register Op0) {
1833switch (RetVT.SimpleTy) {
1834 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0);
1835 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0);
1836 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0);
1837 default: return Register();
1838}
1839}
1840
1841Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Register Op0) {
1842 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1843 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZ256rk, RC: &X86::VR256XRegClass, Op0);
1844 }
1845 return Register();
1846}
1847
1848Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Register Op0) {
1849 if ((Subtarget->hasBWI())) {
1850 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2WZrk, RC: &X86::VR512RegClass, Op0);
1851 }
1852 return Register();
1853}
1854
1855Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, Register Op0) {
1856switch (RetVT.SimpleTy) {
1857 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0);
1858 case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0);
1859 default: return Register();
1860}
1861}
1862
1863Register fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, Register Op0) {
1864 if (RetVT.SimpleTy != MVT::v64i8)
1865 return Register();
1866 if ((Subtarget->hasBWI())) {
1867 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVM2BZrk, RC: &X86::VR512RegClass, Op0);
1868 }
1869 return Register();
1870}
1871
1872Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Register Op0) {
1873 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1874 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWYrr, RC: &X86::VR256RegClass, Op0);
1875 }
1876 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1877 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
1878 }
1879 return Register();
1880}
1881
1882Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Register Op0) {
1883 if ((Subtarget->hasAVX512())) {
1884 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZrr, RC: &X86::VR512RegClass, Op0);
1885 }
1886 return Register();
1887}
1888
1889Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(MVT RetVT, Register Op0) {
1890switch (RetVT.SimpleTy) {
1891 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
1892 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
1893 default: return Register();
1894}
1895}
1896
1897Register fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(MVT RetVT, Register Op0) {
1898 if (RetVT.SimpleTy != MVT::v32i16)
1899 return Register();
1900 if ((Subtarget->hasBWI())) {
1901 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZrr, RC: &X86::VR512RegClass, Op0);
1902 }
1903 return Register();
1904}
1905
1906Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Register Op0) {
1907 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1908 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDYrr, RC: &X86::VR256RegClass, Op0);
1909 }
1910 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1911 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
1912 }
1913 return Register();
1914}
1915
1916Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Register Op0) {
1917 if ((Subtarget->hasAVX512())) {
1918 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZrr, RC: &X86::VR512RegClass, Op0);
1919 }
1920 return Register();
1921}
1922
1923Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(MVT RetVT, Register Op0) {
1924switch (RetVT.SimpleTy) {
1925 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
1926 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
1927 default: return Register();
1928}
1929}
1930
1931Register fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(MVT RetVT, Register Op0) {
1932 if (RetVT.SimpleTy != MVT::v16i32)
1933 return Register();
1934 if ((Subtarget->hasAVX512())) {
1935 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZrr, RC: &X86::VR512RegClass, Op0);
1936 }
1937 return Register();
1938}
1939
1940Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(MVT RetVT, Register Op0) {
1941 if (RetVT.SimpleTy != MVT::v4i64)
1942 return Register();
1943 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1944 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQYrr, RC: &X86::VR256RegClass, Op0);
1945 }
1946 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1947 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
1948 }
1949 return Register();
1950}
1951
1952Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(MVT RetVT, Register Op0) {
1953 if (RetVT.SimpleTy != MVT::v8i64)
1954 return Register();
1955 if ((Subtarget->hasAVX512())) {
1956 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZrr, RC: &X86::VR512RegClass, Op0);
1957 }
1958 return Register();
1959}
1960
1961Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
1962 switch (VT.SimpleTy) {
1963 case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0);
1964 case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0);
1965 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0);
1966 case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0);
1967 case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0);
1968 case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0);
1969 case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0);
1970 case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0);
1971 case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0);
1972 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(RetVT, Op0);
1973 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(RetVT, Op0);
1974 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(RetVT, Op0);
1975 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(RetVT, Op0);
1976 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(RetVT, Op0);
1977 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(RetVT, Op0);
1978 default: return Register();
1979 }
1980}
1981
1982// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
1983
1984Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1985 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1986 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1987 }
1988 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1989 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWrr, RC: &X86::VR128RegClass, Op0);
1990 }
1991 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1992 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
1993 }
1994 return Register();
1995}
1996
1997Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1998 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1999 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
2000 }
2001 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2002 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDrr, RC: &X86::VR128RegClass, Op0);
2003 }
2004 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2005 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
2006 }
2007 return Register();
2008}
2009
2010Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Register Op0) {
2011 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2012 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDYrr, RC: &X86::VR256RegClass, Op0);
2013 }
2014 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2015 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
2016 }
2017 return Register();
2018}
2019
2020Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Register Op0) {
2021 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2022 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
2023 }
2024 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2025 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQrr, RC: &X86::VR128RegClass, Op0);
2026 }
2027 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2028 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
2029 }
2030 return Register();
2031}
2032
2033Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Register Op0) {
2034 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2035 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQYrr, RC: &X86::VR256RegClass, Op0);
2036 }
2037 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2038 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
2039 }
2040 return Register();
2041}
2042
2043Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Register Op0) {
2044 if ((Subtarget->hasAVX512())) {
2045 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXBQZrr, RC: &X86::VR512RegClass, Op0);
2046 }
2047 return Register();
2048}
2049
2050Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, Register Op0) {
2051switch (RetVT.SimpleTy) {
2052 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
2053 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
2054 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
2055 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
2056 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
2057 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
2058 default: return Register();
2059}
2060}
2061
2062Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Register Op0) {
2063 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2064 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
2065 }
2066 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2067 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDrr, RC: &X86::VR128RegClass, Op0);
2068 }
2069 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2070 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
2071 }
2072 return Register();
2073}
2074
2075Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Register Op0) {
2076 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2077 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
2078 }
2079 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2080 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQrr, RC: &X86::VR128RegClass, Op0);
2081 }
2082 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2083 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
2084 }
2085 return Register();
2086}
2087
2088Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Register Op0) {
2089 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2090 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQYrr, RC: &X86::VR256RegClass, Op0);
2091 }
2092 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2093 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
2094 }
2095 return Register();
2096}
2097
2098Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, Register Op0) {
2099switch (RetVT.SimpleTy) {
2100 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
2101 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
2102 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
2103 default: return Register();
2104}
2105}
2106
2107Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, Register Op0) {
2108 if (RetVT.SimpleTy != MVT::v2i64)
2109 return Register();
2110 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2111 return fastEmitInst_r(MachineInstOpcode: X86::PMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
2112 }
2113 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2114 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQrr, RC: &X86::VR128RegClass, Op0);
2115 }
2116 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2117 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
2118 }
2119 return Register();
2120}
2121
2122Register fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, Register Op0) {
2123 switch (VT.SimpleTy) {
2124 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
2125 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
2126 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
2127 default: return Register();
2128 }
2129}
2130
2131// FastEmit functions for ISD::SINT_TO_FP.
2132
2133Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
2134 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2135 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2136 }
2137 return Register();
2138}
2139
2140Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
2141 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2142 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2143 }
2144 return Register();
2145}
2146
2147Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
2148switch (RetVT.SimpleTy) {
2149 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2150 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2151 default: return Register();
2152}
2153}
2154
2155Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
2156 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2157 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2158 }
2159 return Register();
2160}
2161
2162Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
2163 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2164 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2165 }
2166 return Register();
2167}
2168
2169Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
2170switch (RetVT.SimpleTy) {
2171 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2172 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2173 default: return Register();
2174}
2175}
2176
2177Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2178 if (RetVT.SimpleTy != MVT::v8f16)
2179 return Register();
2180 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2181 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2182 }
2183 return Register();
2184}
2185
2186Register fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
2187 if (RetVT.SimpleTy != MVT::v16f16)
2188 return Register();
2189 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2190 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2191 }
2192 return Register();
2193}
2194
2195Register fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
2196 if (RetVT.SimpleTy != MVT::v32f16)
2197 return Register();
2198 if ((Subtarget->hasFP16())) {
2199 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2200 }
2201 return Register();
2202}
2203
2204Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
2205 if ((Subtarget->hasVLX())) {
2206 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2207 }
2208 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2209 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2210 }
2211 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2212 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2213 }
2214 return Register();
2215}
2216
2217Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
2218 if ((Subtarget->hasVLX())) {
2219 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2220 }
2221 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2222 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2223 }
2224 return Register();
2225}
2226
2227Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2228switch (RetVT.SimpleTy) {
2229 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2230 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2231 default: return Register();
2232}
2233}
2234
2235Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
2236 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2237 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2238 }
2239 return Register();
2240}
2241
2242Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
2243 if ((Subtarget->hasVLX())) {
2244 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2245 }
2246 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2247 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2248 }
2249 return Register();
2250}
2251
2252Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
2253 if ((Subtarget->hasAVX512())) {
2254 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2255 }
2256 return Register();
2257}
2258
2259Register fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
2260switch (RetVT.SimpleTy) {
2261 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2262 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2263 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2264 default: return Register();
2265}
2266}
2267
2268Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
2269 if ((Subtarget->hasFP16())) {
2270 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2271 }
2272 return Register();
2273}
2274
2275Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
2276 if ((Subtarget->hasAVX512())) {
2277 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2278 }
2279 return Register();
2280}
2281
2282Register fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
2283switch (RetVT.SimpleTy) {
2284 case MVT::v16f16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2285 case MVT::v16f32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2286 default: return Register();
2287}
2288}
2289
2290Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2291 if (RetVT.SimpleTy != MVT::v2f64)
2292 return Register();
2293 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2294 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2295 }
2296 return Register();
2297}
2298
2299Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
2300 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2301 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
2302 }
2303 return Register();
2304}
2305
2306Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
2307 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2308 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2309 }
2310 return Register();
2311}
2312
2313Register fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
2314switch (RetVT.SimpleTy) {
2315 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2316 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2317 default: return Register();
2318}
2319}
2320
2321Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
2322 if ((Subtarget->hasFP16())) {
2323 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
2324 }
2325 return Register();
2326}
2327
2328Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
2329 if ((Subtarget->hasDQI())) {
2330 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
2331 }
2332 return Register();
2333}
2334
2335Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
2336 if ((Subtarget->hasDQI())) {
2337 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2338 }
2339 return Register();
2340}
2341
2342Register fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
2343switch (RetVT.SimpleTy) {
2344 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
2345 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
2346 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
2347 default: return Register();
2348}
2349}
2350
2351Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
2352 switch (VT.SimpleTy) {
2353 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
2354 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
2355 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2356 case MVT::v16i16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
2357 case MVT::v32i16: return fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
2358 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2359 case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
2360 case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
2361 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
2362 case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
2363 case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
2364 default: return Register();
2365 }
2366}
2367
2368// FastEmit functions for ISD::STRICT_FP_EXTEND.
2369
2370Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2371 if (RetVT.SimpleTy != MVT::f64)
2372 return Register();
2373 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2374 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SDrr, RC: &X86::FR64RegClass, Op0);
2375 }
2376 return Register();
2377}
2378
2379Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Register Op0) {
2380 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2381 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ256rr, RC: &X86::VR256XRegClass, Op0);
2382 }
2383 return Register();
2384}
2385
2386Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Register Op0) {
2387 if ((Subtarget->hasFP16())) {
2388 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrr, RC: &X86::VR512RegClass, Op0);
2389 }
2390 return Register();
2391}
2392
2393Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2394switch (RetVT.SimpleTy) {
2395 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
2396 case MVT::v8f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
2397 default: return Register();
2398}
2399}
2400
2401Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(MVT RetVT, Register Op0) {
2402 if (RetVT.SimpleTy != MVT::v16f32)
2403 return Register();
2404 if ((Subtarget->hasFP16())) {
2405 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrr, RC: &X86::VR512RegClass, Op0);
2406 }
2407 return Register();
2408}
2409
2410Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2411 if (RetVT.SimpleTy != MVT::v4f64)
2412 return Register();
2413 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2414 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2415 }
2416 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2417 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDYrr, RC: &X86::VR256RegClass, Op0);
2418 }
2419 return Register();
2420}
2421
2422Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(MVT RetVT, Register Op0) {
2423 if (RetVT.SimpleTy != MVT::v8f64)
2424 return Register();
2425 if ((Subtarget->hasAVX512())) {
2426 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrr, RC: &X86::VR512RegClass, Op0);
2427 }
2428 return Register();
2429}
2430
2431Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2432 switch (VT.SimpleTy) {
2433 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2434 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
2435 case MVT::v16f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
2436 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
2437 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
2438 default: return Register();
2439 }
2440}
2441
2442// FastEmit functions for ISD::STRICT_FP_ROUND.
2443
2444Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2445 if (RetVT.SimpleTy != MVT::f32)
2446 return Register();
2447 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2448 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SSrr, RC: &X86::FR32RegClass, Op0);
2449 }
2450 return Register();
2451}
2452
2453Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2454 switch (VT.SimpleTy) {
2455 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
2456 default: return Register();
2457 }
2458}
2459
2460// FastEmit functions for ISD::STRICT_FP_TO_SINT.
2461
2462Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
2463 if ((Subtarget->hasFP16())) {
2464 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr, RC: &X86::GR32RegClass, Op0);
2465 }
2466 return Register();
2467}
2468
2469Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
2470 if ((Subtarget->hasFP16())) {
2471 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2472 }
2473 return Register();
2474}
2475
2476Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
2477switch (RetVT.SimpleTy) {
2478 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
2479 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
2480 default: return Register();
2481}
2482}
2483
2484Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
2485 if ((Subtarget->hasAVX512())) {
2486 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr, RC: &X86::GR32RegClass, Op0);
2487 }
2488 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2489 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2490 }
2491 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2492 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr, RC: &X86::GR32RegClass, Op0);
2493 }
2494 return Register();
2495}
2496
2497Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
2498 if ((Subtarget->hasAVX512())) {
2499 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2500 }
2501 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2502 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2503 }
2504 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2505 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr, RC: &X86::GR64RegClass, Op0);
2506 }
2507 return Register();
2508}
2509
2510Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
2511switch (RetVT.SimpleTy) {
2512 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
2513 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
2514 default: return Register();
2515}
2516}
2517
2518Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
2519 if ((Subtarget->hasAVX512())) {
2520 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr, RC: &X86::GR32RegClass, Op0);
2521 }
2522 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2523 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2524 }
2525 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2526 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr, RC: &X86::GR32RegClass, Op0);
2527 }
2528 return Register();
2529}
2530
2531Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
2532 if ((Subtarget->hasAVX512())) {
2533 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr, RC: &X86::GR64RegClass, Op0);
2534 }
2535 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2536 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2537 }
2538 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2539 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr, RC: &X86::GR64RegClass, Op0);
2540 }
2541 return Register();
2542}
2543
2544Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
2545switch (RetVT.SimpleTy) {
2546 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
2547 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
2548 default: return Register();
2549}
2550}
2551
2552Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, Register Op0) {
2553 if (RetVT.SimpleTy != MVT::v4i32)
2554 return Register();
2555 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2556 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
2557 }
2558 return Register();
2559}
2560
2561Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2562 switch (VT.SimpleTy) {
2563 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
2564 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
2565 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
2566 case MVT::v4f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
2567 default: return Register();
2568 }
2569}
2570
2571// FastEmit functions for ISD::STRICT_FP_TO_UINT.
2572
2573Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
2574 if ((Subtarget->hasFP16())) {
2575 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr, RC: &X86::GR32RegClass, Op0);
2576 }
2577 return Register();
2578}
2579
2580Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
2581 if ((Subtarget->hasFP16())) {
2582 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2583 }
2584 return Register();
2585}
2586
2587Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
2588switch (RetVT.SimpleTy) {
2589 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
2590 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
2591 default: return Register();
2592}
2593}
2594
2595Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
2596 if ((Subtarget->hasAVX512())) {
2597 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr, RC: &X86::GR32RegClass, Op0);
2598 }
2599 return Register();
2600}
2601
2602Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
2603 if ((Subtarget->hasAVX512())) {
2604 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2605 }
2606 return Register();
2607}
2608
2609Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
2610switch (RetVT.SimpleTy) {
2611 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
2612 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
2613 default: return Register();
2614}
2615}
2616
2617Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
2618 if ((Subtarget->hasAVX512())) {
2619 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr, RC: &X86::GR32RegClass, Op0);
2620 }
2621 return Register();
2622}
2623
2624Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
2625 if ((Subtarget->hasAVX512())) {
2626 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr, RC: &X86::GR64RegClass, Op0);
2627 }
2628 return Register();
2629}
2630
2631Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
2632switch (RetVT.SimpleTy) {
2633 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
2634 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
2635 default: return Register();
2636}
2637}
2638
2639Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2640 switch (VT.SimpleTy) {
2641 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
2642 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
2643 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
2644 default: return Register();
2645 }
2646}
2647
2648// FastEmit functions for ISD::STRICT_FSQRT.
2649
2650Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
2651 if (RetVT.SimpleTy != MVT::f32)
2652 return Register();
2653 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2654 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSSr, RC: &X86::FR32RegClass, Op0);
2655 }
2656 if ((!Subtarget->hasSSE1())) {
2657 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp32, RC: &X86::RFP32RegClass, Op0);
2658 }
2659 return Register();
2660}
2661
2662Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
2663 if (RetVT.SimpleTy != MVT::f64)
2664 return Register();
2665 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2666 return fastEmitInst_r(MachineInstOpcode: X86::SQRTSDr, RC: &X86::FR64RegClass, Op0);
2667 }
2668 if ((!Subtarget->hasSSE2())) {
2669 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp64, RC: &X86::RFP64RegClass, Op0);
2670 }
2671 return Register();
2672}
2673
2674Register fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(MVT RetVT, Register Op0) {
2675 if (RetVT.SimpleTy != MVT::f80)
2676 return Register();
2677 if ((Subtarget->hasX87())) {
2678 return fastEmitInst_r(MachineInstOpcode: X86::SQRT_Fp80, RC: &X86::RFP80RegClass, Op0);
2679 }
2680 return Register();
2681}
2682
2683Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2684 if (RetVT.SimpleTy != MVT::v8f16)
2685 return Register();
2686 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2687 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
2688 }
2689 return Register();
2690}
2691
2692Register fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(MVT RetVT, Register Op0) {
2693 if (RetVT.SimpleTy != MVT::v16f16)
2694 return Register();
2695 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2696 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
2697 }
2698 return Register();
2699}
2700
2701Register fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(MVT RetVT, Register Op0) {
2702 if (RetVT.SimpleTy != MVT::v32f16)
2703 return Register();
2704 if ((Subtarget->hasFP16())) {
2705 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPHZr, RC: &X86::VR512RegClass, Op0);
2706 }
2707 return Register();
2708}
2709
2710Register fastEmit_ISD_STRICT_FSQRT_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2711 if (RetVT.SimpleTy != MVT::v8bf16)
2712 return Register();
2713 if ((Subtarget->hasAVX10_2())) {
2714 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
2715 }
2716 return Register();
2717}
2718
2719Register fastEmit_ISD_STRICT_FSQRT_MVT_v16bf16_r(MVT RetVT, Register Op0) {
2720 if (RetVT.SimpleTy != MVT::v16bf16)
2721 return Register();
2722 if ((Subtarget->hasAVX10_2())) {
2723 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
2724 }
2725 return Register();
2726}
2727
2728Register fastEmit_ISD_STRICT_FSQRT_MVT_v32bf16_r(MVT RetVT, Register Op0) {
2729 if (RetVT.SimpleTy != MVT::v32bf16)
2730 return Register();
2731 if ((Subtarget->hasAVX10_2())) {
2732 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
2733 }
2734 return Register();
2735}
2736
2737Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2738 if (RetVT.SimpleTy != MVT::v4f32)
2739 return Register();
2740 if ((Subtarget->hasVLX())) {
2741 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ128r, RC: &X86::VR128XRegClass, Op0);
2742 }
2743 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2744 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPSr, RC: &X86::VR128RegClass, Op0);
2745 }
2746 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2747 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSr, RC: &X86::VR128RegClass, Op0);
2748 }
2749 return Register();
2750}
2751
2752Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
2753 if (RetVT.SimpleTy != MVT::v8f32)
2754 return Register();
2755 if ((Subtarget->hasVLX())) {
2756 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZ256r, RC: &X86::VR256XRegClass, Op0);
2757 }
2758 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2759 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSYr, RC: &X86::VR256RegClass, Op0);
2760 }
2761 return Register();
2762}
2763
2764Register fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(MVT RetVT, Register Op0) {
2765 if (RetVT.SimpleTy != MVT::v16f32)
2766 return Register();
2767 if ((Subtarget->hasAVX512())) {
2768 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPSZr, RC: &X86::VR512RegClass, Op0);
2769 }
2770 return Register();
2771}
2772
2773Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2774 if (RetVT.SimpleTy != MVT::v2f64)
2775 return Register();
2776 if ((Subtarget->hasVLX())) {
2777 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ128r, RC: &X86::VR128XRegClass, Op0);
2778 }
2779 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2780 return fastEmitInst_r(MachineInstOpcode: X86::SQRTPDr, RC: &X86::VR128RegClass, Op0);
2781 }
2782 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2783 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDr, RC: &X86::VR128RegClass, Op0);
2784 }
2785 return Register();
2786}
2787
2788Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(MVT RetVT, Register Op0) {
2789 if (RetVT.SimpleTy != MVT::v4f64)
2790 return Register();
2791 if ((Subtarget->hasVLX())) {
2792 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZ256r, RC: &X86::VR256XRegClass, Op0);
2793 }
2794 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2795 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDYr, RC: &X86::VR256RegClass, Op0);
2796 }
2797 return Register();
2798}
2799
2800Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(MVT RetVT, Register Op0) {
2801 if (RetVT.SimpleTy != MVT::v8f64)
2802 return Register();
2803 if ((Subtarget->hasAVX512())) {
2804 return fastEmitInst_r(MachineInstOpcode: X86::VSQRTPDZr, RC: &X86::VR512RegClass, Op0);
2805 }
2806 return Register();
2807}
2808
2809Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
2810 switch (VT.SimpleTy) {
2811 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
2812 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
2813 case MVT::f80: return fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(RetVT, Op0);
2814 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
2815 case MVT::v16f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(RetVT, Op0);
2816 case MVT::v32f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(RetVT, Op0);
2817 case MVT::v8bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8bf16_r(RetVT, Op0);
2818 case MVT::v16bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16bf16_r(RetVT, Op0);
2819 case MVT::v32bf16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32bf16_r(RetVT, Op0);
2820 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
2821 case MVT::v8f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(RetVT, Op0);
2822 case MVT::v16f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(RetVT, Op0);
2823 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
2824 case MVT::v4f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(RetVT, Op0);
2825 case MVT::v8f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(RetVT, Op0);
2826 default: return Register();
2827 }
2828}
2829
2830// FastEmit functions for ISD::STRICT_SINT_TO_FP.
2831
2832Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
2833 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2834 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SSrr, RC: &X86::FR32RegClass, Op0);
2835 }
2836 return Register();
2837}
2838
2839Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
2840 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2841 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI2SDrr, RC: &X86::FR64RegClass, Op0);
2842 }
2843 return Register();
2844}
2845
2846Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
2847switch (RetVT.SimpleTy) {
2848 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2849 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2850 default: return Register();
2851}
2852}
2853
2854Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
2855 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2856 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SSrr, RC: &X86::FR32RegClass, Op0);
2857 }
2858 return Register();
2859}
2860
2861Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
2862 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2863 return fastEmitInst_r(MachineInstOpcode: X86::CVTSI642SDrr, RC: &X86::FR64RegClass, Op0);
2864 }
2865 return Register();
2866}
2867
2868Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
2869switch (RetVT.SimpleTy) {
2870 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2871 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2872 default: return Register();
2873}
2874}
2875
2876Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2877 if (RetVT.SimpleTy != MVT::v8f16)
2878 return Register();
2879 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2880 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
2881 }
2882 return Register();
2883}
2884
2885Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
2886 if (RetVT.SimpleTy != MVT::v16f16)
2887 return Register();
2888 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2889 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
2890 }
2891 return Register();
2892}
2893
2894Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
2895 if (RetVT.SimpleTy != MVT::v32f16)
2896 return Register();
2897 if ((Subtarget->hasFP16())) {
2898 return fastEmitInst_r(MachineInstOpcode: X86::VCVTW2PHZrr, RC: &X86::VR512RegClass, Op0);
2899 }
2900 return Register();
2901}
2902
2903Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
2904 if ((Subtarget->hasVLX())) {
2905 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
2906 }
2907 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2908 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2909 }
2910 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2911 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSrr, RC: &X86::VR128RegClass, Op0);
2912 }
2913 return Register();
2914}
2915
2916Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
2917 if ((Subtarget->hasVLX())) {
2918 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
2919 }
2920 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2921 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDYrr, RC: &X86::VR256RegClass, Op0);
2922 }
2923 return Register();
2924}
2925
2926Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2927switch (RetVT.SimpleTy) {
2928 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2929 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2930 default: return Register();
2931}
2932}
2933
2934Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
2935 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2936 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
2937 }
2938 return Register();
2939}
2940
2941Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
2942 if ((Subtarget->hasVLX())) {
2943 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
2944 }
2945 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2946 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSYrr, RC: &X86::VR256RegClass, Op0);
2947 }
2948 return Register();
2949}
2950
2951Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
2952 if ((Subtarget->hasAVX512())) {
2953 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
2954 }
2955 return Register();
2956}
2957
2958Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
2959switch (RetVT.SimpleTy) {
2960 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2961 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2962 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2963 default: return Register();
2964}
2965}
2966
2967Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
2968 if ((Subtarget->hasFP16())) {
2969 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
2970 }
2971 return Register();
2972}
2973
2974Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
2975 if ((Subtarget->hasAVX512())) {
2976 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
2977 }
2978 return Register();
2979}
2980
2981Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
2982switch (RetVT.SimpleTy) {
2983 case MVT::v16f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2984 case MVT::v16f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2985 default: return Register();
2986}
2987}
2988
2989Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2990 if (RetVT.SimpleTy != MVT::v2f64)
2991 return Register();
2992 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2993 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
2994 }
2995 return Register();
2996}
2997
2998Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
2999 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3000 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3001 }
3002 return Register();
3003}
3004
3005Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3006 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3007 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3008 }
3009 return Register();
3010}
3011
3012Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3013switch (RetVT.SimpleTy) {
3014 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3015 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3016 default: return Register();
3017}
3018}
3019
3020Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3021 if ((Subtarget->hasFP16())) {
3022 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3023 }
3024 return Register();
3025}
3026
3027Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3028 if ((Subtarget->hasDQI())) {
3029 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3030 }
3031 return Register();
3032}
3033
3034Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3035 if ((Subtarget->hasDQI())) {
3036 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3037 }
3038 return Register();
3039}
3040
3041Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3042switch (RetVT.SimpleTy) {
3043 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3044 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3045 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3046 default: return Register();
3047}
3048}
3049
3050Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3051 switch (VT.SimpleTy) {
3052 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
3053 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
3054 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3055 case MVT::v16i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3056 case MVT::v32i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3057 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3058 case MVT::v8i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3059 case MVT::v16i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3060 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3061 case MVT::v4i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3062 case MVT::v8i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3063 default: return Register();
3064 }
3065}
3066
3067// FastEmit functions for ISD::STRICT_UINT_TO_FP.
3068
3069Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3070 if (RetVT.SimpleTy != MVT::v8f16)
3071 return Register();
3072 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3073 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3074 }
3075 return Register();
3076}
3077
3078Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
3079 if (RetVT.SimpleTy != MVT::v16f16)
3080 return Register();
3081 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3082 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3083 }
3084 return Register();
3085}
3086
3087Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
3088 if (RetVT.SimpleTy != MVT::v32f16)
3089 return Register();
3090 if ((Subtarget->hasFP16())) {
3091 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3092 }
3093 return Register();
3094}
3095
3096Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
3097 if ((Subtarget->hasVLX())) {
3098 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3099 }
3100 return Register();
3101}
3102
3103Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
3104 if ((Subtarget->hasVLX())) {
3105 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3106 }
3107 return Register();
3108}
3109
3110Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3111switch (RetVT.SimpleTy) {
3112 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3113 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3114 default: return Register();
3115}
3116}
3117
3118Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
3119 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3120 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3121 }
3122 return Register();
3123}
3124
3125Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
3126 if ((Subtarget->hasVLX())) {
3127 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3128 }
3129 return Register();
3130}
3131
3132Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
3133 if ((Subtarget->hasAVX512())) {
3134 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3135 }
3136 return Register();
3137}
3138
3139Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
3140switch (RetVT.SimpleTy) {
3141 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3142 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3143 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3144 default: return Register();
3145}
3146}
3147
3148Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
3149 if ((Subtarget->hasFP16())) {
3150 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3151 }
3152 return Register();
3153}
3154
3155Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
3156 if ((Subtarget->hasAVX512())) {
3157 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3158 }
3159 return Register();
3160}
3161
3162Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
3163switch (RetVT.SimpleTy) {
3164 case MVT::v16f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3165 case MVT::v16f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3166 default: return Register();
3167}
3168}
3169
3170Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
3171 if (RetVT.SimpleTy != MVT::v2f64)
3172 return Register();
3173 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3174 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3175 }
3176 return Register();
3177}
3178
3179Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
3180 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3181 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3182 }
3183 return Register();
3184}
3185
3186Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3187 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3188 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3189 }
3190 return Register();
3191}
3192
3193Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3194switch (RetVT.SimpleTy) {
3195 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3196 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3197 default: return Register();
3198}
3199}
3200
3201Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3202 if ((Subtarget->hasFP16())) {
3203 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3204 }
3205 return Register();
3206}
3207
3208Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3209 if ((Subtarget->hasDQI())) {
3210 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3211 }
3212 return Register();
3213}
3214
3215Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3216 if ((Subtarget->hasDQI())) {
3217 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3218 }
3219 return Register();
3220}
3221
3222Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3223switch (RetVT.SimpleTy) {
3224 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3225 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3226 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3227 default: return Register();
3228}
3229}
3230
3231Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3232 switch (VT.SimpleTy) {
3233 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3234 case MVT::v16i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3235 case MVT::v32i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3236 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3237 case MVT::v8i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3238 case MVT::v16i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3239 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3240 case MVT::v4i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3241 case MVT::v8i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3242 default: return Register();
3243 }
3244}
3245
3246// FastEmit functions for ISD::TRUNCATE.
3247
3248Register fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, Register Op0) {
3249 if (RetVT.SimpleTy != MVT::i8)
3250 return Register();
3251 if ((Subtarget->is64Bit())) {
3252 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: X86::sub_8bit);
3253 }
3254 return Register();
3255}
3256
3257Register fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Register Op0) {
3258 if ((Subtarget->is64Bit())) {
3259 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3260 }
3261 return Register();
3262}
3263
3264Register fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Register Op0) {
3265 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3266}
3267
3268Register fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, Register Op0) {
3269switch (RetVT.SimpleTy) {
3270 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0);
3271 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0);
3272 default: return Register();
3273}
3274}
3275
3276Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Register Op0) {
3277 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0, Idx: X86::sub_8bit);
3278}
3279
3280Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Register Op0) {
3281 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0, Idx: X86::sub_16bit);
3282}
3283
3284Register fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Register Op0) {
3285 return fastEmitInst_extractsubreg(RetVT: MVT::i32, Op0, Idx: X86::sub_32bit);
3286}
3287
3288Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
3289switch (RetVT.SimpleTy) {
3290 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0);
3291 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0);
3292 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0);
3293 default: return Register();
3294}
3295}
3296
3297Register fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, Register Op0) {
3298 if (RetVT.SimpleTy != MVT::v16i8)
3299 return Register();
3300 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3301 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ256rr, RC: &X86::VR128XRegClass, Op0);
3302 }
3303 return Register();
3304}
3305
3306Register fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, Register Op0) {
3307 if (RetVT.SimpleTy != MVT::v32i8)
3308 return Register();
3309 if ((Subtarget->hasBWI())) {
3310 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZrr, RC: &X86::VR256XRegClass, Op0);
3311 }
3312 return Register();
3313}
3314
3315Register fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, Register Op0) {
3316 if (RetVT.SimpleTy != MVT::v8i16)
3317 return Register();
3318 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3319 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ256rr, RC: &X86::VR128XRegClass, Op0);
3320 }
3321 return Register();
3322}
3323
3324Register fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Register Op0) {
3325 if ((Subtarget->hasAVX512())) {
3326 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZrr, RC: &X86::VR128XRegClass, Op0);
3327 }
3328 return Register();
3329}
3330
3331Register fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Register Op0) {
3332 if ((Subtarget->hasAVX512())) {
3333 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZrr, RC: &X86::VR256XRegClass, Op0);
3334 }
3335 return Register();
3336}
3337
3338Register fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, Register Op0) {
3339switch (RetVT.SimpleTy) {
3340 case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0);
3341 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0);
3342 default: return Register();
3343}
3344}
3345
3346Register fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, Register Op0) {
3347 if (RetVT.SimpleTy != MVT::v4i32)
3348 return Register();
3349 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3350 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ256rr, RC: &X86::VR128XRegClass, Op0);
3351 }
3352 return Register();
3353}
3354
3355Register fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Register Op0) {
3356 if ((Subtarget->hasAVX512())) {
3357 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZrr, RC: &X86::VR128XRegClass, Op0);
3358 }
3359 return Register();
3360}
3361
3362Register fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Register Op0) {
3363 if ((Subtarget->hasAVX512())) {
3364 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZrr, RC: &X86::VR256XRegClass, Op0);
3365 }
3366 return Register();
3367}
3368
3369Register fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, Register Op0) {
3370switch (RetVT.SimpleTy) {
3371 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0);
3372 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0);
3373 default: return Register();
3374}
3375}
3376
3377Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3378 switch (VT.SimpleTy) {
3379 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0);
3380 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0);
3381 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
3382 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0);
3383 case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0);
3384 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0);
3385 case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0);
3386 case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0);
3387 case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0);
3388 default: return Register();
3389 }
3390}
3391
3392// FastEmit functions for ISD::UINT_TO_FP.
3393
3394Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3395 if (RetVT.SimpleTy != MVT::v8f16)
3396 return Register();
3397 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3398 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
3399 }
3400 return Register();
3401}
3402
3403Register fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, Register Op0) {
3404 if (RetVT.SimpleTy != MVT::v16f16)
3405 return Register();
3406 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3407 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZ256rr, RC: &X86::VR256XRegClass, Op0);
3408 }
3409 return Register();
3410}
3411
3412Register fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, Register Op0) {
3413 if (RetVT.SimpleTy != MVT::v32f16)
3414 return Register();
3415 if ((Subtarget->hasFP16())) {
3416 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUW2PHZrr, RC: &X86::VR512RegClass, Op0);
3417 }
3418 return Register();
3419}
3420
3421Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Register Op0) {
3422 if ((Subtarget->hasVLX())) {
3423 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
3424 }
3425 return Register();
3426}
3427
3428Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Register Op0) {
3429 if ((Subtarget->hasVLX())) {
3430 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3431 }
3432 return Register();
3433}
3434
3435Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3436switch (RetVT.SimpleTy) {
3437 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3438 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3439 default: return Register();
3440}
3441}
3442
3443Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Register Op0) {
3444 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3445 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
3446 }
3447 return Register();
3448}
3449
3450Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Register Op0) {
3451 if ((Subtarget->hasVLX())) {
3452 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
3453 }
3454 return Register();
3455}
3456
3457Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Register Op0) {
3458 if ((Subtarget->hasAVX512())) {
3459 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3460 }
3461 return Register();
3462}
3463
3464Register fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, Register Op0) {
3465switch (RetVT.SimpleTy) {
3466 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3467 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3468 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3469 default: return Register();
3470}
3471}
3472
3473Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Register Op0) {
3474 if ((Subtarget->hasFP16())) {
3475 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZrr, RC: &X86::VR256XRegClass, Op0);
3476 }
3477 return Register();
3478}
3479
3480Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Register Op0) {
3481 if ((Subtarget->hasAVX512())) {
3482 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PSZrr, RC: &X86::VR512RegClass, Op0);
3483 }
3484 return Register();
3485}
3486
3487Register fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, Register Op0) {
3488switch (RetVT.SimpleTy) {
3489 case MVT::v16f16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3490 case MVT::v16f32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3491 default: return Register();
3492}
3493}
3494
3495Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
3496 if (RetVT.SimpleTy != MVT::v2f64)
3497 return Register();
3498 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3499 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
3500 }
3501 return Register();
3502}
3503
3504Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Register Op0) {
3505 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3506 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
3507 }
3508 return Register();
3509}
3510
3511Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Register Op0) {
3512 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3513 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
3514 }
3515 return Register();
3516}
3517
3518Register fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, Register Op0) {
3519switch (RetVT.SimpleTy) {
3520 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3521 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3522 default: return Register();
3523}
3524}
3525
3526Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Register Op0) {
3527 if ((Subtarget->hasFP16())) {
3528 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZrr, RC: &X86::VR128XRegClass, Op0);
3529 }
3530 return Register();
3531}
3532
3533Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Register Op0) {
3534 if ((Subtarget->hasDQI())) {
3535 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZrr, RC: &X86::VR256XRegClass, Op0);
3536 }
3537 return Register();
3538}
3539
3540Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Register Op0) {
3541 if ((Subtarget->hasDQI())) {
3542 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PDZrr, RC: &X86::VR512RegClass, Op0);
3543 }
3544 return Register();
3545}
3546
3547Register fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, Register Op0) {
3548switch (RetVT.SimpleTy) {
3549 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3550 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3551 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3552 default: return Register();
3553}
3554}
3555
3556Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3557 switch (VT.SimpleTy) {
3558 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3559 case MVT::v16i16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3560 case MVT::v32i16: return fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3561 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3562 case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3563 case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3564 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3565 case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3566 case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3567 default: return Register();
3568 }
3569}
3570
3571// FastEmit functions for ISD::ZERO_EXTEND.
3572
3573Register fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, Register Op0) {
3574 if (RetVT.SimpleTy != MVT::i32)
3575 return Register();
3576 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr8, RC: &X86::GR32RegClass, Op0);
3577}
3578
3579Register fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, Register Op0) {
3580 if (RetVT.SimpleTy != MVT::i32)
3581 return Register();
3582 return fastEmitInst_r(MachineInstOpcode: X86::MOVZX32rr16, RC: &X86::GR32RegClass, Op0);
3583}
3584
3585Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Register Op0) {
3586 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3587 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWYrr, RC: &X86::VR256RegClass, Op0);
3588 }
3589 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3590 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ256rr, RC: &X86::VR256XRegClass, Op0);
3591 }
3592 return Register();
3593}
3594
3595Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Register Op0) {
3596 if ((Subtarget->hasAVX512())) {
3597 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZrr, RC: &X86::VR512RegClass, Op0);
3598 }
3599 return Register();
3600}
3601
3602Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(MVT RetVT, Register Op0) {
3603switch (RetVT.SimpleTy) {
3604 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
3605 case MVT::v16i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
3606 default: return Register();
3607}
3608}
3609
3610Register fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(MVT RetVT, Register Op0) {
3611 if (RetVT.SimpleTy != MVT::v32i16)
3612 return Register();
3613 if ((Subtarget->hasBWI())) {
3614 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZrr, RC: &X86::VR512RegClass, Op0);
3615 }
3616 return Register();
3617}
3618
3619Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Register Op0) {
3620 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3621 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDYrr, RC: &X86::VR256RegClass, Op0);
3622 }
3623 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3624 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ256rr, RC: &X86::VR256XRegClass, Op0);
3625 }
3626 return Register();
3627}
3628
3629Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Register Op0) {
3630 if ((Subtarget->hasAVX512())) {
3631 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZrr, RC: &X86::VR512RegClass, Op0);
3632 }
3633 return Register();
3634}
3635
3636Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(MVT RetVT, Register Op0) {
3637switch (RetVT.SimpleTy) {
3638 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
3639 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
3640 default: return Register();
3641}
3642}
3643
3644Register fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(MVT RetVT, Register Op0) {
3645 if (RetVT.SimpleTy != MVT::v16i32)
3646 return Register();
3647 if ((Subtarget->hasAVX512())) {
3648 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZrr, RC: &X86::VR512RegClass, Op0);
3649 }
3650 return Register();
3651}
3652
3653Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(MVT RetVT, Register Op0) {
3654 if (RetVT.SimpleTy != MVT::v4i64)
3655 return Register();
3656 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3657 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQYrr, RC: &X86::VR256RegClass, Op0);
3658 }
3659 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3660 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ256rr, RC: &X86::VR256XRegClass, Op0);
3661 }
3662 return Register();
3663}
3664
3665Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(MVT RetVT, Register Op0) {
3666 if (RetVT.SimpleTy != MVT::v8i64)
3667 return Register();
3668 if ((Subtarget->hasAVX512())) {
3669 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZrr, RC: &X86::VR512RegClass, Op0);
3670 }
3671 return Register();
3672}
3673
3674Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3675 switch (VT.SimpleTy) {
3676 case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0);
3677 case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0);
3678 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(RetVT, Op0);
3679 case MVT::v32i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(RetVT, Op0);
3680 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(RetVT, Op0);
3681 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(RetVT, Op0);
3682 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(RetVT, Op0);
3683 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(RetVT, Op0);
3684 default: return Register();
3685 }
3686}
3687
3688// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
3689
3690Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Register Op0) {
3691 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3692 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3693 }
3694 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3695 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWrr, RC: &X86::VR128RegClass, Op0);
3696 }
3697 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3698 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBWZ128rr, RC: &X86::VR128XRegClass, Op0);
3699 }
3700 return Register();
3701}
3702
3703Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Register Op0) {
3704 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3705 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3706 }
3707 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3708 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDrr, RC: &X86::VR128RegClass, Op0);
3709 }
3710 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3711 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ128rr, RC: &X86::VR128XRegClass, Op0);
3712 }
3713 return Register();
3714}
3715
3716Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Register Op0) {
3717 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3718 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDYrr, RC: &X86::VR256RegClass, Op0);
3719 }
3720 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3721 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBDZ256rr, RC: &X86::VR256XRegClass, Op0);
3722 }
3723 return Register();
3724}
3725
3726Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Register Op0) {
3727 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3728 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3729 }
3730 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3731 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQrr, RC: &X86::VR128RegClass, Op0);
3732 }
3733 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3734 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ128rr, RC: &X86::VR128XRegClass, Op0);
3735 }
3736 return Register();
3737}
3738
3739Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Register Op0) {
3740 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3741 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQYrr, RC: &X86::VR256RegClass, Op0);
3742 }
3743 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3744 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZ256rr, RC: &X86::VR256XRegClass, Op0);
3745 }
3746 return Register();
3747}
3748
3749Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Register Op0) {
3750 if ((Subtarget->hasAVX512())) {
3751 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXBQZrr, RC: &X86::VR512RegClass, Op0);
3752 }
3753 return Register();
3754}
3755
3756Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, Register Op0) {
3757switch (RetVT.SimpleTy) {
3758 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
3759 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
3760 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
3761 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
3762 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
3763 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
3764 default: return Register();
3765}
3766}
3767
3768Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Register Op0) {
3769 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3770 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3771 }
3772 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3773 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDrr, RC: &X86::VR128RegClass, Op0);
3774 }
3775 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3776 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWDZ128rr, RC: &X86::VR128XRegClass, Op0);
3777 }
3778 return Register();
3779}
3780
3781Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Register Op0) {
3782 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3783 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3784 }
3785 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3786 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQrr, RC: &X86::VR128RegClass, Op0);
3787 }
3788 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3789 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ128rr, RC: &X86::VR128XRegClass, Op0);
3790 }
3791 return Register();
3792}
3793
3794Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Register Op0) {
3795 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3796 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQYrr, RC: &X86::VR256RegClass, Op0);
3797 }
3798 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3799 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXWQZ256rr, RC: &X86::VR256XRegClass, Op0);
3800 }
3801 return Register();
3802}
3803
3804Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, Register Op0) {
3805switch (RetVT.SimpleTy) {
3806 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
3807 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
3808 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
3809 default: return Register();
3810}
3811}
3812
3813Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, Register Op0) {
3814 if (RetVT.SimpleTy != MVT::v2i64)
3815 return Register();
3816 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3817 return fastEmitInst_r(MachineInstOpcode: X86::PMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3818 }
3819 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3820 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQrr, RC: &X86::VR128RegClass, Op0);
3821 }
3822 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3823 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVZXDQZ128rr, RC: &X86::VR128XRegClass, Op0);
3824 }
3825 return Register();
3826}
3827
3828Register fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, Register Op0) {
3829 switch (VT.SimpleTy) {
3830 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
3831 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
3832 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
3833 default: return Register();
3834 }
3835}
3836
3837// FastEmit functions for X86ISD::CALL.
3838
3839Register fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, Register Op0) {
3840 if (RetVT.SimpleTy != MVT::isVoid)
3841 return Register();
3842 if ((!Subtarget->is64Bit())) {
3843 return fastEmitInst_r(MachineInstOpcode: X86::CALL16r, RC: &X86::GR16RegClass, Op0);
3844 }
3845 return Register();
3846}
3847
3848Register fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
3849 if (RetVT.SimpleTy != MVT::isVoid)
3850 return Register();
3851 if ((!Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3852 return fastEmitInst_r(MachineInstOpcode: X86::INDIRECT_THUNK_CALL32, RC: &X86::GR32RegClass, Op0);
3853 }
3854 if ((!Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3855 return fastEmitInst_r(MachineInstOpcode: X86::CALL32r, RC: &X86::GR32RegClass, Op0);
3856 }
3857 return Register();
3858}
3859
3860Register fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
3861 if (RetVT.SimpleTy != MVT::isVoid)
3862 return Register();
3863 if ((MF->getFunction().getParent()->getModuleFlag(Key: "import-call-optimization")) && (Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3864 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r_ImpCall, RC: &X86::GR64_ARegClass, Op0);
3865 }
3866 if ((Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3867 return fastEmitInst_r(MachineInstOpcode: X86::INDIRECT_THUNK_CALL64, RC: &X86::GR64RegClass, Op0);
3868 }
3869 if ((!MF->getFunction().getParent()->getModuleFlag(Key: "import-call-optimization")) && (Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3870 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r, RC: &X86::GR64RegClass, Op0);
3871 }
3872 return Register();
3873}
3874
3875Register fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
3876 switch (VT.SimpleTy) {
3877 case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0);
3878 case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0);
3879 case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0);
3880 default: return Register();
3881 }
3882}
3883
3884// FastEmit functions for X86ISD::CONFLICT.
3885
3886Register fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, Register Op0) {
3887 if (RetVT.SimpleTy != MVT::v4i32)
3888 return Register();
3889 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3890 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ128rr, RC: &X86::VR128XRegClass, Op0);
3891 }
3892 return Register();
3893}
3894
3895Register fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, Register Op0) {
3896 if (RetVT.SimpleTy != MVT::v8i32)
3897 return Register();
3898 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3899 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZ256rr, RC: &X86::VR256XRegClass, Op0);
3900 }
3901 return Register();
3902}
3903
3904Register fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, Register Op0) {
3905 if (RetVT.SimpleTy != MVT::v16i32)
3906 return Register();
3907 if ((Subtarget->hasCDI())) {
3908 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTDZrr, RC: &X86::VR512RegClass, Op0);
3909 }
3910 return Register();
3911}
3912
3913Register fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, Register Op0) {
3914 if (RetVT.SimpleTy != MVT::v2i64)
3915 return Register();
3916 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3917 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ128rr, RC: &X86::VR128XRegClass, Op0);
3918 }
3919 return Register();
3920}
3921
3922Register fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, Register Op0) {
3923 if (RetVT.SimpleTy != MVT::v4i64)
3924 return Register();
3925 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3926 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZ256rr, RC: &X86::VR256XRegClass, Op0);
3927 }
3928 return Register();
3929}
3930
3931Register fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, Register Op0) {
3932 if (RetVT.SimpleTy != MVT::v8i64)
3933 return Register();
3934 if ((Subtarget->hasCDI())) {
3935 return fastEmitInst_r(MachineInstOpcode: X86::VPCONFLICTQZrr, RC: &X86::VR512RegClass, Op0);
3936 }
3937 return Register();
3938}
3939
3940Register fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, Register Op0) {
3941 switch (VT.SimpleTy) {
3942 case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0);
3943 case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0);
3944 case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0);
3945 case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0);
3946 case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0);
3947 case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0);
3948 default: return Register();
3949 }
3950}
3951
3952// FastEmit functions for X86ISD::CVTNEPS2BF16.
3953
3954Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(MVT RetVT, Register Op0) {
3955 if (RetVT.SimpleTy != MVT::v8bf16)
3956 return Register();
3957 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3958 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0);
3959 }
3960 if ((Subtarget->hasAVXNECONVERT())) {
3961 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16rr, RC: &X86::VR128RegClass, Op0);
3962 }
3963 return Register();
3964}
3965
3966Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(MVT RetVT, Register Op0) {
3967 if (RetVT.SimpleTy != MVT::v8bf16)
3968 return Register();
3969 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3970 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
3971 }
3972 return Register();
3973}
3974
3975Register fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(MVT RetVT, Register Op0) {
3976 if (RetVT.SimpleTy != MVT::v16bf16)
3977 return Register();
3978 if ((Subtarget->hasBF16())) {
3979 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
3980 }
3981 return Register();
3982}
3983
3984Register fastEmit_X86ISD_CVTNEPS2BF16_r(MVT VT, MVT RetVT, Register Op0) {
3985 switch (VT.SimpleTy) {
3986 case MVT::v4f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(RetVT, Op0);
3987 case MVT::v8f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(RetVT, Op0);
3988 case MVT::v16f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(RetVT, Op0);
3989 default: return Register();
3990 }
3991}
3992
3993// FastEmit functions for X86ISD::CVTP2IBS.
3994
3995Register fastEmit_X86ISD_CVTP2IBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
3996 if (RetVT.SimpleTy != MVT::v8i16)
3997 return Register();
3998 if ((Subtarget->hasAVX10_2())) {
3999 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4000 }
4001 return Register();
4002}
4003
4004Register fastEmit_X86ISD_CVTP2IBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4005 if (RetVT.SimpleTy != MVT::v16i16)
4006 return Register();
4007 if ((Subtarget->hasAVX10_2())) {
4008 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4009 }
4010 return Register();
4011}
4012
4013Register fastEmit_X86ISD_CVTP2IBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4014 if (RetVT.SimpleTy != MVT::v32i16)
4015 return Register();
4016 if ((Subtarget->hasAVX10_2())) {
4017 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IBSZrr, RC: &X86::VR512RegClass, Op0);
4018 }
4019 return Register();
4020}
4021
4022Register fastEmit_X86ISD_CVTP2IBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
4023 if (RetVT.SimpleTy != MVT::v8i16)
4024 return Register();
4025 if ((Subtarget->hasAVX10_2())) {
4026 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4027 }
4028 return Register();
4029}
4030
4031Register fastEmit_X86ISD_CVTP2IBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
4032 if (RetVT.SimpleTy != MVT::v16i16)
4033 return Register();
4034 if ((Subtarget->hasAVX10_2())) {
4035 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4036 }
4037 return Register();
4038}
4039
4040Register fastEmit_X86ISD_CVTP2IBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4041 if (RetVT.SimpleTy != MVT::v32i16)
4042 return Register();
4043 if ((Subtarget->hasAVX10_2())) {
4044 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IBSZrr, RC: &X86::VR512RegClass, Op0);
4045 }
4046 return Register();
4047}
4048
4049Register fastEmit_X86ISD_CVTP2IBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4050 if (RetVT.SimpleTy != MVT::v4i32)
4051 return Register();
4052 if ((Subtarget->hasAVX10_2())) {
4053 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4054 }
4055 return Register();
4056}
4057
4058Register fastEmit_X86ISD_CVTP2IBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
4059 if (RetVT.SimpleTy != MVT::v8i32)
4060 return Register();
4061 if ((Subtarget->hasAVX10_2())) {
4062 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4063 }
4064 return Register();
4065}
4066
4067Register fastEmit_X86ISD_CVTP2IBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
4068 if (RetVT.SimpleTy != MVT::v16i32)
4069 return Register();
4070 if ((Subtarget->hasAVX10_2())) {
4071 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IBSZrr, RC: &X86::VR512RegClass, Op0);
4072 }
4073 return Register();
4074}
4075
4076Register fastEmit_X86ISD_CVTP2IBS_r(MVT VT, MVT RetVT, Register Op0) {
4077 switch (VT.SimpleTy) {
4078 case MVT::v8f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v8f16_r(RetVT, Op0);
4079 case MVT::v16f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v16f16_r(RetVT, Op0);
4080 case MVT::v32f16: return fastEmit_X86ISD_CVTP2IBS_MVT_v32f16_r(RetVT, Op0);
4081 case MVT::v8bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v8bf16_r(RetVT, Op0);
4082 case MVT::v16bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v16bf16_r(RetVT, Op0);
4083 case MVT::v32bf16: return fastEmit_X86ISD_CVTP2IBS_MVT_v32bf16_r(RetVT, Op0);
4084 case MVT::v4f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v4f32_r(RetVT, Op0);
4085 case MVT::v8f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v8f32_r(RetVT, Op0);
4086 case MVT::v16f32: return fastEmit_X86ISD_CVTP2IBS_MVT_v16f32_r(RetVT, Op0);
4087 default: return Register();
4088 }
4089}
4090
4091// FastEmit functions for X86ISD::CVTP2IUBS.
4092
4093Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
4094 if (RetVT.SimpleTy != MVT::v8i16)
4095 return Register();
4096 if ((Subtarget->hasAVX10_2())) {
4097 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4098 }
4099 return Register();
4100}
4101
4102Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4103 if (RetVT.SimpleTy != MVT::v16i16)
4104 return Register();
4105 if ((Subtarget->hasAVX10_2())) {
4106 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4107 }
4108 return Register();
4109}
4110
4111Register fastEmit_X86ISD_CVTP2IUBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4112 if (RetVT.SimpleTy != MVT::v32i16)
4113 return Register();
4114 if ((Subtarget->hasAVX10_2())) {
4115 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2IUBSZrr, RC: &X86::VR512RegClass, Op0);
4116 }
4117 return Register();
4118}
4119
4120Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
4121 if (RetVT.SimpleTy != MVT::v8i16)
4122 return Register();
4123 if ((Subtarget->hasAVX10_2())) {
4124 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4125 }
4126 return Register();
4127}
4128
4129Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
4130 if (RetVT.SimpleTy != MVT::v16i16)
4131 return Register();
4132 if ((Subtarget->hasAVX10_2())) {
4133 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4134 }
4135 return Register();
4136}
4137
4138Register fastEmit_X86ISD_CVTP2IUBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
4139 if (RetVT.SimpleTy != MVT::v32i16)
4140 return Register();
4141 if ((Subtarget->hasAVX10_2())) {
4142 return fastEmitInst_r(MachineInstOpcode: X86::VCVTBF162IUBSZrr, RC: &X86::VR512RegClass, Op0);
4143 }
4144 return Register();
4145}
4146
4147Register fastEmit_X86ISD_CVTP2IUBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
4148 if (RetVT.SimpleTy != MVT::v4i32)
4149 return Register();
4150 if ((Subtarget->hasAVX10_2())) {
4151 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4152 }
4153 return Register();
4154}
4155
4156Register fastEmit_X86ISD_CVTP2IUBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
4157 if (RetVT.SimpleTy != MVT::v8i32)
4158 return Register();
4159 if ((Subtarget->hasAVX10_2())) {
4160 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4161 }
4162 return Register();
4163}
4164
4165Register fastEmit_X86ISD_CVTP2IUBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
4166 if (RetVT.SimpleTy != MVT::v16i32)
4167 return Register();
4168 if ((Subtarget->hasAVX10_2())) {
4169 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2IUBSZrr, RC: &X86::VR512RegClass, Op0);
4170 }
4171 return Register();
4172}
4173
4174Register fastEmit_X86ISD_CVTP2IUBS_r(MVT VT, MVT RetVT, Register Op0) {
4175 switch (VT.SimpleTy) {
4176 case MVT::v8f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8f16_r(RetVT, Op0);
4177 case MVT::v16f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16f16_r(RetVT, Op0);
4178 case MVT::v32f16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v32f16_r(RetVT, Op0);
4179 case MVT::v8bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8bf16_r(RetVT, Op0);
4180 case MVT::v16bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16bf16_r(RetVT, Op0);
4181 case MVT::v32bf16: return fastEmit_X86ISD_CVTP2IUBS_MVT_v32bf16_r(RetVT, Op0);
4182 case MVT::v4f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v4f32_r(RetVT, Op0);
4183 case MVT::v8f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v8f32_r(RetVT, Op0);
4184 case MVT::v16f32: return fastEmit_X86ISD_CVTP2IUBS_MVT_v16f32_r(RetVT, Op0);
4185 default: return Register();
4186 }
4187}
4188
4189// FastEmit functions for X86ISD::CVTP2SI.
4190
4191Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
4192 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4193 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
4194 }
4195 return Register();
4196}
4197
4198Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
4199 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4200 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4201 }
4202 return Register();
4203}
4204
4205Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
4206 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4207 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4208 }
4209 return Register();
4210}
4211
4212Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
4213 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4214 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4215 }
4216 return Register();
4217}
4218
4219Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
4220 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4221 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4222 }
4223 return Register();
4224}
4225
4226Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
4227 if ((Subtarget->hasFP16())) {
4228 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
4229 }
4230 return Register();
4231}
4232
4233Register fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4234switch (RetVT.SimpleTy) {
4235 case MVT::v8i16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
4236 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
4237 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
4238 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
4239 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
4240 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
4241 default: return Register();
4242}
4243}
4244
4245Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
4246 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4247 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
4248 }
4249 return Register();
4250}
4251
4252Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
4253 if ((Subtarget->hasFP16())) {
4254 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
4255 }
4256 return Register();
4257}
4258
4259Register fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
4260switch (RetVT.SimpleTy) {
4261 case MVT::v16i16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
4262 case MVT::v16i32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
4263 default: return Register();
4264}
4265}
4266
4267Register fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
4268 if (RetVT.SimpleTy != MVT::v32i16)
4269 return Register();
4270 if ((Subtarget->hasFP16())) {
4271 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2WZrr, RC: &X86::VR512RegClass, Op0);
4272 }
4273 return Register();
4274}
4275
4276Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
4277 if ((Subtarget->hasVLX())) {
4278 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4279 }
4280 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4281 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4282 }
4283 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4284 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQrr, RC: &X86::VR128RegClass, Op0);
4285 }
4286 return Register();
4287}
4288
4289Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
4290 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4291 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4292 }
4293 return Register();
4294}
4295
4296Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
4297 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4298 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4299 }
4300 return Register();
4301}
4302
4303Register fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4304switch (RetVT.SimpleTy) {
4305 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
4306 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
4307 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
4308 default: return Register();
4309}
4310}
4311
4312Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
4313 if ((Subtarget->hasVLX())) {
4314 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
4315 }
4316 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4317 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
4318 }
4319 return Register();
4320}
4321
4322Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
4323 if ((Subtarget->hasDQI())) {
4324 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
4325 }
4326 return Register();
4327}
4328
4329Register fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
4330switch (RetVT.SimpleTy) {
4331 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
4332 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
4333 default: return Register();
4334}
4335}
4336
4337Register fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
4338 if (RetVT.SimpleTy != MVT::v16i32)
4339 return Register();
4340 if ((Subtarget->hasAVX512())) {
4341 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
4342 }
4343 return Register();
4344}
4345
4346Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
4347 if ((Subtarget->hasVLX())) {
4348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
4349 }
4350 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4351 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4352 }
4353 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4354 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQrr, RC: &X86::VR128RegClass, Op0);
4355 }
4356 return Register();
4357}
4358
4359Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
4360 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4361 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
4362 }
4363 return Register();
4364}
4365
4366Register fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4367switch (RetVT.SimpleTy) {
4368 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
4369 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
4370 default: return Register();
4371}
4372}
4373
4374Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
4375 if ((Subtarget->hasVLX())) {
4376 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
4377 }
4378 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4379 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
4380 }
4381 return Register();
4382}
4383
4384Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
4385 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4386 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
4387 }
4388 return Register();
4389}
4390
4391Register fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
4392switch (RetVT.SimpleTy) {
4393 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
4394 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
4395 default: return Register();
4396}
4397}
4398
4399Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
4400 if ((Subtarget->hasAVX512())) {
4401 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
4402 }
4403 return Register();
4404}
4405
4406Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
4407 if ((Subtarget->hasDQI())) {
4408 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
4409 }
4410 return Register();
4411}
4412
4413Register fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
4414switch (RetVT.SimpleTy) {
4415 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
4416 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
4417 default: return Register();
4418}
4419}
4420
4421Register fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
4422 switch (VT.SimpleTy) {
4423 case MVT::v8f16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(RetVT, Op0);
4424 case MVT::v16f16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(RetVT, Op0);
4425 case MVT::v32f16: return fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(RetVT, Op0);
4426 case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0);
4427 case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0);
4428 case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0);
4429 case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0);
4430 case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0);
4431 case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0);
4432 default: return Register();
4433 }
4434}
4435
4436// FastEmit functions for X86ISD::CVTP2UI.
4437
4438Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
4439 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4440 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
4441 }
4442 return Register();
4443}
4444
4445Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
4446 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4447 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4448 }
4449 return Register();
4450}
4451
4452Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
4453 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4455 }
4456 return Register();
4457}
4458
4459Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
4460 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4462 }
4463 return Register();
4464}
4465
4466Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
4467 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4469 }
4470 return Register();
4471}
4472
4473Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
4474 if ((Subtarget->hasFP16())) {
4475 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
4476 }
4477 return Register();
4478}
4479
4480Register fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4481switch (RetVT.SimpleTy) {
4482 case MVT::v8i16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
4483 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
4484 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
4485 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
4486 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
4487 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
4488 default: return Register();
4489}
4490}
4491
4492Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
4493 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
4495 }
4496 return Register();
4497}
4498
4499Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
4500 if ((Subtarget->hasFP16())) {
4501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
4502 }
4503 return Register();
4504}
4505
4506Register fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
4507switch (RetVT.SimpleTy) {
4508 case MVT::v16i16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
4509 case MVT::v16i32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
4510 default: return Register();
4511}
4512}
4513
4514Register fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
4515 if (RetVT.SimpleTy != MVT::v32i16)
4516 return Register();
4517 if ((Subtarget->hasFP16())) {
4518 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
4519 }
4520 return Register();
4521}
4522
4523Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
4524 if ((Subtarget->hasVLX())) {
4525 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4526 }
4527 return Register();
4528}
4529
4530Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
4531 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4532 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4533 }
4534 return Register();
4535}
4536
4537Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
4538 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4539 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4540 }
4541 return Register();
4542}
4543
4544Register fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4545switch (RetVT.SimpleTy) {
4546 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
4547 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
4548 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
4549 default: return Register();
4550}
4551}
4552
4553Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
4554 if ((Subtarget->hasVLX())) {
4555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
4556 }
4557 return Register();
4558}
4559
4560Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
4561 if ((Subtarget->hasDQI())) {
4562 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
4563 }
4564 return Register();
4565}
4566
4567Register fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
4568switch (RetVT.SimpleTy) {
4569 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
4570 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
4571 default: return Register();
4572}
4573}
4574
4575Register fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
4576 if (RetVT.SimpleTy != MVT::v16i32)
4577 return Register();
4578 if ((Subtarget->hasAVX512())) {
4579 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
4580 }
4581 return Register();
4582}
4583
4584Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
4585 if ((Subtarget->hasVLX())) {
4586 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
4587 }
4588 return Register();
4589}
4590
4591Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
4592 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4593 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
4594 }
4595 return Register();
4596}
4597
4598Register fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4599switch (RetVT.SimpleTy) {
4600 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
4601 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
4602 default: return Register();
4603}
4604}
4605
4606Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
4607 if ((Subtarget->hasVLX())) {
4608 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
4609 }
4610 return Register();
4611}
4612
4613Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
4614 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4615 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
4616 }
4617 return Register();
4618}
4619
4620Register fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
4621switch (RetVT.SimpleTy) {
4622 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
4623 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
4624 default: return Register();
4625}
4626}
4627
4628Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
4629 if ((Subtarget->hasAVX512())) {
4630 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
4631 }
4632 return Register();
4633}
4634
4635Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
4636 if ((Subtarget->hasDQI())) {
4637 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
4638 }
4639 return Register();
4640}
4641
4642Register fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
4643switch (RetVT.SimpleTy) {
4644 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
4645 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
4646 default: return Register();
4647}
4648}
4649
4650Register fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
4651 switch (VT.SimpleTy) {
4652 case MVT::v8f16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(RetVT, Op0);
4653 case MVT::v16f16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(RetVT, Op0);
4654 case MVT::v32f16: return fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(RetVT, Op0);
4655 case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0);
4656 case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0);
4657 case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0);
4658 case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0);
4659 case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0);
4660 case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0);
4661 default: return Register();
4662 }
4663}
4664
4665// FastEmit functions for X86ISD::CVTPH2PS.
4666
4667Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Register Op0) {
4668 if ((Subtarget->hasVLX())) {
4669 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4670 }
4671 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4672 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
4673 }
4674 return Register();
4675}
4676
4677Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Register Op0) {
4678 if ((Subtarget->hasVLX())) {
4679 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
4680 }
4681 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4682 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
4683 }
4684 return Register();
4685}
4686
4687Register fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, Register Op0) {
4688switch (RetVT.SimpleTy) {
4689 case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
4690 case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
4691 default: return Register();
4692}
4693}
4694
4695Register fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, Register Op0) {
4696 if (RetVT.SimpleTy != MVT::v16f32)
4697 return Register();
4698 if ((Subtarget->hasAVX512())) {
4699 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
4700 }
4701 return Register();
4702}
4703
4704Register fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, Register Op0) {
4705 switch (VT.SimpleTy) {
4706 case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
4707 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
4708 default: return Register();
4709 }
4710}
4711
4712// FastEmit functions for X86ISD::CVTPH2PS_SAE.
4713
4714Register fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(MVT RetVT, Register Op0) {
4715 if (RetVT.SimpleTy != MVT::v16f32)
4716 return Register();
4717 if ((Subtarget->hasAVX512())) {
4718 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrrb, RC: &X86::VR512RegClass, Op0);
4719 }
4720 return Register();
4721}
4722
4723Register fastEmit_X86ISD_CVTPH2PS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
4724 switch (VT.SimpleTy) {
4725 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(RetVT, Op0);
4726 default: return Register();
4727 }
4728}
4729
4730// FastEmit functions for X86ISD::CVTS2SI.
4731
4732Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Register Op0) {
4733 if ((Subtarget->hasFP16())) {
4734 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4735 }
4736 return Register();
4737}
4738
4739Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Register Op0) {
4740 if ((Subtarget->hasFP16())) {
4741 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4742 }
4743 return Register();
4744}
4745
4746Register fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4747switch (RetVT.SimpleTy) {
4748 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Op0);
4749 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Op0);
4750 default: return Register();
4751}
4752}
4753
4754Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Register Op0) {
4755 if ((Subtarget->hasAVX512())) {
4756 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4757 }
4758 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4759 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4760 }
4761 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4762 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4763 }
4764 return Register();
4765}
4766
4767Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Register Op0) {
4768 if ((Subtarget->hasAVX512())) {
4769 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4770 }
4771 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4772 return fastEmitInst_r(MachineInstOpcode: X86::CVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4773 }
4774 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4775 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4776 }
4777 return Register();
4778}
4779
4780Register fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4781switch (RetVT.SimpleTy) {
4782 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0);
4783 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0);
4784 default: return Register();
4785}
4786}
4787
4788Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Register Op0) {
4789 if ((Subtarget->hasAVX512())) {
4790 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
4791 }
4792 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4793 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4794 }
4795 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4796 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
4797 }
4798 return Register();
4799}
4800
4801Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Register Op0) {
4802 if ((Subtarget->hasAVX512())) {
4803 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4804 }
4805 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4806 return fastEmitInst_r(MachineInstOpcode: X86::CVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4807 }
4808 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4809 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
4810 }
4811 return Register();
4812}
4813
4814Register fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4815switch (RetVT.SimpleTy) {
4816 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0);
4817 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0);
4818 default: return Register();
4819}
4820}
4821
4822Register fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, Register Op0) {
4823 switch (VT.SimpleTy) {
4824 case MVT::v8f16: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(RetVT, Op0);
4825 case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0);
4826 case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0);
4827 default: return Register();
4828 }
4829}
4830
4831// FastEmit functions for X86ISD::CVTS2UI.
4832
4833Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Register Op0) {
4834 if ((Subtarget->hasFP16())) {
4835 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4836 }
4837 return Register();
4838}
4839
4840Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Register Op0) {
4841 if ((Subtarget->hasFP16())) {
4842 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4843 }
4844 return Register();
4845}
4846
4847Register fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
4848switch (RetVT.SimpleTy) {
4849 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Op0);
4850 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Op0);
4851 default: return Register();
4852}
4853}
4854
4855Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Register Op0) {
4856 if ((Subtarget->hasAVX512())) {
4857 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4858 }
4859 return Register();
4860}
4861
4862Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Register Op0) {
4863 if ((Subtarget->hasAVX512())) {
4864 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4865 }
4866 return Register();
4867}
4868
4869Register fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
4870switch (RetVT.SimpleTy) {
4871 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0);
4872 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0);
4873 default: return Register();
4874}
4875}
4876
4877Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Register Op0) {
4878 if ((Subtarget->hasAVX512())) {
4879 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
4880 }
4881 return Register();
4882}
4883
4884Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Register Op0) {
4885 if ((Subtarget->hasAVX512())) {
4886 return fastEmitInst_r(MachineInstOpcode: X86::VCVTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
4887 }
4888 return Register();
4889}
4890
4891Register fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
4892switch (RetVT.SimpleTy) {
4893 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0);
4894 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0);
4895 default: return Register();
4896}
4897}
4898
4899Register fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, Register Op0) {
4900 switch (VT.SimpleTy) {
4901 case MVT::v8f16: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(RetVT, Op0);
4902 case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0);
4903 case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0);
4904 default: return Register();
4905 }
4906}
4907
4908// FastEmit functions for X86ISD::CVTSI2P.
4909
4910Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
4911 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4912 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4913 }
4914 return Register();
4915}
4916
4917Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
4918 if ((Subtarget->hasVLX())) {
4919 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
4920 }
4921 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4922 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4923 }
4924 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4925 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
4926 }
4927 return Register();
4928}
4929
4930Register fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
4931switch (RetVT.SimpleTy) {
4932 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
4933 case MVT::v2f64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
4934 default: return Register();
4935}
4936}
4937
4938Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
4939 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4940 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
4941 }
4942 return Register();
4943}
4944
4945Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
4946 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4947 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
4948 }
4949 return Register();
4950}
4951
4952Register fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
4953switch (RetVT.SimpleTy) {
4954 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
4955 case MVT::v4f32: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
4956 default: return Register();
4957}
4958}
4959
4960Register fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
4961 if (RetVT.SimpleTy != MVT::v8f16)
4962 return Register();
4963 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4964 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
4965 }
4966 return Register();
4967}
4968
4969Register fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, Register Op0) {
4970 switch (VT.SimpleTy) {
4971 case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
4972 case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
4973 case MVT::v4i64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
4974 default: return Register();
4975 }
4976}
4977
4978// FastEmit functions for X86ISD::CVTTP2IBS.
4979
4980Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
4981 if (RetVT.SimpleTy != MVT::v8i16)
4982 return Register();
4983 if ((Subtarget->hasAVX10_2())) {
4984 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
4985 }
4986 return Register();
4987}
4988
4989Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
4990 if (RetVT.SimpleTy != MVT::v16i16)
4991 return Register();
4992 if ((Subtarget->hasAVX10_2())) {
4993 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
4994 }
4995 return Register();
4996}
4997
4998Register fastEmit_X86ISD_CVTTP2IBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
4999 if (RetVT.SimpleTy != MVT::v32i16)
5000 return Register();
5001 if ((Subtarget->hasAVX10_2())) {
5002 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZrr, RC: &X86::VR512RegClass, Op0);
5003 }
5004 return Register();
5005}
5006
5007Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
5008 if (RetVT.SimpleTy != MVT::v8i16)
5009 return Register();
5010 if ((Subtarget->hasAVX10_2())) {
5011 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5012 }
5013 return Register();
5014}
5015
5016Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
5017 if (RetVT.SimpleTy != MVT::v16i16)
5018 return Register();
5019 if ((Subtarget->hasAVX10_2())) {
5020 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5021 }
5022 return Register();
5023}
5024
5025Register fastEmit_X86ISD_CVTTP2IBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
5026 if (RetVT.SimpleTy != MVT::v32i16)
5027 return Register();
5028 if ((Subtarget->hasAVX10_2())) {
5029 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IBSZrr, RC: &X86::VR512RegClass, Op0);
5030 }
5031 return Register();
5032}
5033
5034Register fastEmit_X86ISD_CVTTP2IBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5035 if (RetVT.SimpleTy != MVT::v4i32)
5036 return Register();
5037 if ((Subtarget->hasAVX10_2())) {
5038 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5039 }
5040 return Register();
5041}
5042
5043Register fastEmit_X86ISD_CVTTP2IBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5044 if (RetVT.SimpleTy != MVT::v8i32)
5045 return Register();
5046 if ((Subtarget->hasAVX10_2())) {
5047 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5048 }
5049 return Register();
5050}
5051
5052Register fastEmit_X86ISD_CVTTP2IBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5053 if (RetVT.SimpleTy != MVT::v16i32)
5054 return Register();
5055 if ((Subtarget->hasAVX10_2())) {
5056 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZrr, RC: &X86::VR512RegClass, Op0);
5057 }
5058 return Register();
5059}
5060
5061Register fastEmit_X86ISD_CVTTP2IBS_r(MVT VT, MVT RetVT, Register Op0) {
5062 switch (VT.SimpleTy) {
5063 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8f16_r(RetVT, Op0);
5064 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16f16_r(RetVT, Op0);
5065 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v32f16_r(RetVT, Op0);
5066 case MVT::v8bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8bf16_r(RetVT, Op0);
5067 case MVT::v16bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16bf16_r(RetVT, Op0);
5068 case MVT::v32bf16: return fastEmit_X86ISD_CVTTP2IBS_MVT_v32bf16_r(RetVT, Op0);
5069 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v4f32_r(RetVT, Op0);
5070 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v8f32_r(RetVT, Op0);
5071 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IBS_MVT_v16f32_r(RetVT, Op0);
5072 default: return Register();
5073 }
5074}
5075
5076// FastEmit functions for X86ISD::CVTTP2IBS_SAE.
5077
5078Register fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5079 if (RetVT.SimpleTy != MVT::v32i16)
5080 return Register();
5081 if ((Subtarget->hasAVX10_2())) {
5082 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IBSZrrb, RC: &X86::VR512RegClass, Op0);
5083 }
5084 return Register();
5085}
5086
5087Register fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5088 if (RetVT.SimpleTy != MVT::v16i32)
5089 return Register();
5090 if ((Subtarget->hasAVX10_2())) {
5091 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IBSZrrb, RC: &X86::VR512RegClass, Op0);
5092 }
5093 return Register();
5094}
5095
5096Register fastEmit_X86ISD_CVTTP2IBS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5097 switch (VT.SimpleTy) {
5098 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v32f16_r(RetVT, Op0);
5099 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IBS_SAE_MVT_v16f32_r(RetVT, Op0);
5100 default: return Register();
5101 }
5102}
5103
5104// FastEmit functions for X86ISD::CVTTP2IUBS.
5105
5106Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f16_r(MVT RetVT, Register Op0) {
5107 if (RetVT.SimpleTy != MVT::v8i16)
5108 return Register();
5109 if ((Subtarget->hasAVX10_2())) {
5110 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5111 }
5112 return Register();
5113}
5114
5115Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f16_r(MVT RetVT, Register Op0) {
5116 if (RetVT.SimpleTy != MVT::v16i16)
5117 return Register();
5118 if ((Subtarget->hasAVX10_2())) {
5119 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5120 }
5121 return Register();
5122}
5123
5124Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v32f16_r(MVT RetVT, Register Op0) {
5125 if (RetVT.SimpleTy != MVT::v32i16)
5126 return Register();
5127 if ((Subtarget->hasAVX10_2())) {
5128 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZrr, RC: &X86::VR512RegClass, Op0);
5129 }
5130 return Register();
5131}
5132
5133Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
5134 if (RetVT.SimpleTy != MVT::v8i16)
5135 return Register();
5136 if ((Subtarget->hasAVX10_2())) {
5137 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5138 }
5139 return Register();
5140}
5141
5142Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16bf16_r(MVT RetVT, Register Op0) {
5143 if (RetVT.SimpleTy != MVT::v16i16)
5144 return Register();
5145 if ((Subtarget->hasAVX10_2())) {
5146 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5147 }
5148 return Register();
5149}
5150
5151Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v32bf16_r(MVT RetVT, Register Op0) {
5152 if (RetVT.SimpleTy != MVT::v32i16)
5153 return Register();
5154 if ((Subtarget->hasAVX10_2())) {
5155 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTBF162IUBSZrr, RC: &X86::VR512RegClass, Op0);
5156 }
5157 return Register();
5158}
5159
5160Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5161 if (RetVT.SimpleTy != MVT::v4i32)
5162 return Register();
5163 if ((Subtarget->hasAVX10_2())) {
5164 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZ128rr, RC: &X86::VR128XRegClass, Op0);
5165 }
5166 return Register();
5167}
5168
5169Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5170 if (RetVT.SimpleTy != MVT::v8i32)
5171 return Register();
5172 if ((Subtarget->hasAVX10_2())) {
5173 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZ256rr, RC: &X86::VR256XRegClass, Op0);
5174 }
5175 return Register();
5176}
5177
5178Register fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5179 if (RetVT.SimpleTy != MVT::v16i32)
5180 return Register();
5181 if ((Subtarget->hasAVX10_2())) {
5182 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZrr, RC: &X86::VR512RegClass, Op0);
5183 }
5184 return Register();
5185}
5186
5187Register fastEmit_X86ISD_CVTTP2IUBS_r(MVT VT, MVT RetVT, Register Op0) {
5188 switch (VT.SimpleTy) {
5189 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f16_r(RetVT, Op0);
5190 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f16_r(RetVT, Op0);
5191 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v32f16_r(RetVT, Op0);
5192 case MVT::v8bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8bf16_r(RetVT, Op0);
5193 case MVT::v16bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16bf16_r(RetVT, Op0);
5194 case MVT::v32bf16: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v32bf16_r(RetVT, Op0);
5195 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v4f32_r(RetVT, Op0);
5196 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v8f32_r(RetVT, Op0);
5197 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IUBS_MVT_v16f32_r(RetVT, Op0);
5198 default: return Register();
5199 }
5200}
5201
5202// FastEmit functions for X86ISD::CVTTP2IUBS_SAE.
5203
5204Register fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5205 if (RetVT.SimpleTy != MVT::v32i16)
5206 return Register();
5207 if ((Subtarget->hasAVX10_2())) {
5208 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2IUBSZrrb, RC: &X86::VR512RegClass, Op0);
5209 }
5210 return Register();
5211}
5212
5213Register fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5214 if (RetVT.SimpleTy != MVT::v16i32)
5215 return Register();
5216 if ((Subtarget->hasAVX10_2())) {
5217 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2IUBSZrrb, RC: &X86::VR512RegClass, Op0);
5218 }
5219 return Register();
5220}
5221
5222Register fastEmit_X86ISD_CVTTP2IUBS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5223 switch (VT.SimpleTy) {
5224 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v32f16_r(RetVT, Op0);
5225 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2IUBS_SAE_MVT_v16f32_r(RetVT, Op0);
5226 default: return Register();
5227 }
5228}
5229
5230// FastEmit functions for X86ISD::CVTTP2SI.
5231
5232Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
5233 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5234 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
5235 }
5236 return Register();
5237}
5238
5239Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
5240 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5241 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5242 }
5243 return Register();
5244}
5245
5246Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
5247 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5248 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
5249 }
5250 return Register();
5251}
5252
5253Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
5254 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5255 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5256 }
5257 return Register();
5258}
5259
5260Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
5261 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5262 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5263 }
5264 return Register();
5265}
5266
5267Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
5268 if ((Subtarget->hasFP16())) {
5269 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
5270 }
5271 return Register();
5272}
5273
5274Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
5275switch (RetVT.SimpleTy) {
5276 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
5277 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
5278 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
5279 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
5280 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
5281 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
5282 default: return Register();
5283}
5284}
5285
5286Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
5287 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5288 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
5289 }
5290 return Register();
5291}
5292
5293Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
5294 if ((Subtarget->hasFP16())) {
5295 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
5296 }
5297 return Register();
5298}
5299
5300Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
5301switch (RetVT.SimpleTy) {
5302 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
5303 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
5304 default: return Register();
5305}
5306}
5307
5308Register fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
5309 if (RetVT.SimpleTy != MVT::v32i16)
5310 return Register();
5311 if ((Subtarget->hasFP16())) {
5312 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
5313 }
5314 return Register();
5315}
5316
5317Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5318 if ((Subtarget->hasVLX())) {
5319 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5320 }
5321 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5322 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
5323 }
5324 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5325 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
5326 }
5327 return Register();
5328}
5329
5330Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5331 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5332 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5333 }
5334 return Register();
5335}
5336
5337Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5338 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5339 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5340 }
5341 return Register();
5342}
5343
5344Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
5345switch (RetVT.SimpleTy) {
5346 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
5347 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
5348 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
5349 default: return Register();
5350}
5351}
5352
5353Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5354 if ((Subtarget->hasVLX())) {
5355 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
5356 }
5357 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5358 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
5359 }
5360 return Register();
5361}
5362
5363Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5364 if ((Subtarget->hasDQI())) {
5365 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
5366 }
5367 return Register();
5368}
5369
5370Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
5371switch (RetVT.SimpleTy) {
5372 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
5373 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
5374 default: return Register();
5375}
5376}
5377
5378Register fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
5379 if (RetVT.SimpleTy != MVT::v16i32)
5380 return Register();
5381 if ((Subtarget->hasAVX512())) {
5382 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
5383 }
5384 return Register();
5385}
5386
5387Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5388 if ((Subtarget->hasVLX())) {
5389 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
5390 }
5391 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5392 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
5393 }
5394 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5395 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
5396 }
5397 return Register();
5398}
5399
5400Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5401 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5402 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
5403 }
5404 return Register();
5405}
5406
5407Register fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
5408switch (RetVT.SimpleTy) {
5409 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
5410 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
5411 default: return Register();
5412}
5413}
5414
5415Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5416 if ((Subtarget->hasVLX())) {
5417 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
5418 }
5419 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5420 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
5421 }
5422 return Register();
5423}
5424
5425Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5426 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5427 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
5428 }
5429 return Register();
5430}
5431
5432Register fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
5433switch (RetVT.SimpleTy) {
5434 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
5435 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
5436 default: return Register();
5437}
5438}
5439
5440Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5441 if ((Subtarget->hasAVX512())) {
5442 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
5443 }
5444 return Register();
5445}
5446
5447Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5448 if ((Subtarget->hasDQI())) {
5449 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
5450 }
5451 return Register();
5452}
5453
5454Register fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
5455switch (RetVT.SimpleTy) {
5456 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
5457 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
5458 default: return Register();
5459}
5460}
5461
5462Register fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
5463 switch (VT.SimpleTy) {
5464 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
5465 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
5466 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
5467 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
5468 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
5469 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
5470 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
5471 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
5472 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
5473 default: return Register();
5474 }
5475}
5476
5477// FastEmit functions for X86ISD::CVTTP2SIS.
5478
5479Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5480 if ((Subtarget->hasAVX10_2())) {
5481 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5482 }
5483 return Register();
5484}
5485
5486Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5487 if ((Subtarget->hasAVX10_2())) {
5488 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5489 }
5490 return Register();
5491}
5492
5493Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5494 if ((Subtarget->hasAVX10_2())) {
5495 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5496 }
5497 return Register();
5498}
5499
5500Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
5501switch (RetVT.SimpleTy) {
5502 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i32_r(Op0);
5503 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v2i64_r(Op0);
5504 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_MVT_v4i64_r(Op0);
5505 default: return Register();
5506}
5507}
5508
5509Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5510 if ((Subtarget->hasAVX10_2())) {
5511 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5512 }
5513 return Register();
5514}
5515
5516Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5517 if ((Subtarget->hasAVX10_2())) {
5518 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZrr, RC: &X86::VR512RegClass, Op0);
5519 }
5520 return Register();
5521}
5522
5523Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_r(MVT RetVT, Register Op0) {
5524switch (RetVT.SimpleTy) {
5525 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i32_r(Op0);
5526 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_MVT_v8i64_r(Op0);
5527 default: return Register();
5528}
5529}
5530
5531Register fastEmit_X86ISD_CVTTP2SIS_MVT_v16f32_r(MVT RetVT, Register Op0) {
5532 if (RetVT.SimpleTy != MVT::v16i32)
5533 return Register();
5534 if ((Subtarget->hasAVX10_2())) {
5535 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZrr, RC: &X86::VR512RegClass, Op0);
5536 }
5537 return Register();
5538}
5539
5540Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5541 if ((Subtarget->hasAVX10_2())) {
5542 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5543 }
5544 return Register();
5545}
5546
5547Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5548 if ((Subtarget->hasAVX10_2())) {
5549 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
5550 }
5551 return Register();
5552}
5553
5554Register fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
5555switch (RetVT.SimpleTy) {
5556 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v4i32_r(Op0);
5557 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_MVT_v2i64_r(Op0);
5558 default: return Register();
5559}
5560}
5561
5562Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5563 if ((Subtarget->hasAVX10_2())) {
5564 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ256rr, RC: &X86::VR128XRegClass, Op0);
5565 }
5566 return Register();
5567}
5568
5569Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5570 if ((Subtarget->hasAVX10_2())) {
5571 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ256rr, RC: &X86::VR256XRegClass, Op0);
5572 }
5573 return Register();
5574}
5575
5576Register fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_r(MVT RetVT, Register Op0) {
5577switch (RetVT.SimpleTy) {
5578 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i32_r(Op0);
5579 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_MVT_v4i64_r(Op0);
5580 default: return Register();
5581}
5582}
5583
5584Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5585 if ((Subtarget->hasAVX10_2())) {
5586 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZrr, RC: &X86::VR256XRegClass, Op0);
5587 }
5588 return Register();
5589}
5590
5591Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5592 if ((Subtarget->hasAVX10_2())) {
5593 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZrr, RC: &X86::VR512RegClass, Op0);
5594 }
5595 return Register();
5596}
5597
5598Register fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_r(MVT RetVT, Register Op0) {
5599switch (RetVT.SimpleTy) {
5600 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i32_r(Op0);
5601 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_MVT_v8i64_r(Op0);
5602 default: return Register();
5603}
5604}
5605
5606Register fastEmit_X86ISD_CVTTP2SIS_r(MVT VT, MVT RetVT, Register Op0) {
5607 switch (VT.SimpleTy) {
5608 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f32_r(RetVT, Op0);
5609 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f32_r(RetVT, Op0);
5610 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SIS_MVT_v16f32_r(RetVT, Op0);
5611 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v2f64_r(RetVT, Op0);
5612 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v4f64_r(RetVT, Op0);
5613 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SIS_MVT_v8f64_r(RetVT, Op0);
5614 default: return Register();
5615 }
5616}
5617
5618// FastEmit functions for X86ISD::CVTTP2SIS_SAE.
5619
5620Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
5621 if (RetVT.SimpleTy != MVT::v4i64)
5622 return Register();
5623 if ((Subtarget->hasAVX10_2())) {
5624 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
5625 }
5626 return Register();
5627}
5628
5629Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
5630 if (RetVT.SimpleTy != MVT::v8i64)
5631 return Register();
5632 if ((Subtarget->hasAVX10_2())) {
5633 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZrrb, RC: &X86::VR512RegClass, Op0);
5634 }
5635 return Register();
5636}
5637
5638Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5639 if (RetVT.SimpleTy != MVT::v16i32)
5640 return Register();
5641 if ((Subtarget->hasAVX10_2())) {
5642 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQSZrrb, RC: &X86::VR512RegClass, Op0);
5643 }
5644 return Register();
5645}
5646
5647Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5648 if ((Subtarget->hasAVX10_2())) {
5649 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ256rrb, RC: &X86::VR128XRegClass, Op0);
5650 }
5651 return Register();
5652}
5653
5654Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5655 if ((Subtarget->hasAVX10_2())) {
5656 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
5657 }
5658 return Register();
5659}
5660
5661Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_r(MVT RetVT, Register Op0) {
5662switch (RetVT.SimpleTy) {
5663 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i32_r(Op0);
5664 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_MVT_v4i64_r(Op0);
5665 default: return Register();
5666}
5667}
5668
5669Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5670 if ((Subtarget->hasAVX10_2())) {
5671 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZrrb, RC: &X86::VR256XRegClass, Op0);
5672 }
5673 return Register();
5674}
5675
5676Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5677 if ((Subtarget->hasAVX10_2())) {
5678 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQSZrrb, RC: &X86::VR512RegClass, Op0);
5679 }
5680 return Register();
5681}
5682
5683Register fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
5684switch (RetVT.SimpleTy) {
5685 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5686 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5687 default: return Register();
5688}
5689}
5690
5691Register fastEmit_X86ISD_CVTTP2SIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5692 switch (VT.SimpleTy) {
5693 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f32_r(RetVT, Op0);
5694 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f32_r(RetVT, Op0);
5695 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v16f32_r(RetVT, Op0);
5696 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v4f64_r(RetVT, Op0);
5697 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SIS_SAE_MVT_v8f64_r(RetVT, Op0);
5698 default: return Register();
5699 }
5700}
5701
5702// FastEmit functions for X86ISD::CVTTP2SI_SAE.
5703
5704Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
5705 if (RetVT.SimpleTy != MVT::v8i64)
5706 return Register();
5707 if ((Subtarget->hasFP16())) {
5708 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrrb, RC: &X86::VR512RegClass, Op0);
5709 }
5710 return Register();
5711}
5712
5713Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
5714 if (RetVT.SimpleTy != MVT::v16i32)
5715 return Register();
5716 if ((Subtarget->hasFP16())) {
5717 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrrb, RC: &X86::VR512RegClass, Op0);
5718 }
5719 return Register();
5720}
5721
5722Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
5723 if (RetVT.SimpleTy != MVT::v32i16)
5724 return Register();
5725 if ((Subtarget->hasFP16())) {
5726 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrrb, RC: &X86::VR512RegClass, Op0);
5727 }
5728 return Register();
5729}
5730
5731Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
5732 if (RetVT.SimpleTy != MVT::v8i64)
5733 return Register();
5734 if ((Subtarget->hasDQI())) {
5735 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrrb, RC: &X86::VR512RegClass, Op0);
5736 }
5737 return Register();
5738}
5739
5740Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
5741 if (RetVT.SimpleTy != MVT::v16i32)
5742 return Register();
5743 if ((Subtarget->hasAVX512())) {
5744 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrrb, RC: &X86::VR512RegClass, Op0);
5745 }
5746 return Register();
5747}
5748
5749Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5750 if ((Subtarget->hasAVX512())) {
5751 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrrb, RC: &X86::VR256XRegClass, Op0);
5752 }
5753 return Register();
5754}
5755
5756Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5757 if ((Subtarget->hasDQI())) {
5758 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrrb, RC: &X86::VR512RegClass, Op0);
5759 }
5760 return Register();
5761}
5762
5763Register fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
5764switch (RetVT.SimpleTy) {
5765 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5766 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5767 default: return Register();
5768}
5769}
5770
5771Register fastEmit_X86ISD_CVTTP2SI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
5772 switch (VT.SimpleTy) {
5773 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(RetVT, Op0);
5774 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(RetVT, Op0);
5775 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(RetVT, Op0);
5776 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(RetVT, Op0);
5777 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(RetVT, Op0);
5778 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(RetVT, Op0);
5779 default: return Register();
5780 }
5781}
5782
5783// FastEmit functions for X86ISD::CVTTP2UI.
5784
5785Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
5786 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5787 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
5788 }
5789 return Register();
5790}
5791
5792Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
5793 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5794 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5795 }
5796 return Register();
5797}
5798
5799Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
5800 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5801 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5802 }
5803 return Register();
5804}
5805
5806Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
5807 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5808 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5809 }
5810 return Register();
5811}
5812
5813Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
5814 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5815 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5816 }
5817 return Register();
5818}
5819
5820Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
5821 if ((Subtarget->hasFP16())) {
5822 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
5823 }
5824 return Register();
5825}
5826
5827Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
5828switch (RetVT.SimpleTy) {
5829 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
5830 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
5831 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
5832 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
5833 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
5834 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
5835 default: return Register();
5836}
5837}
5838
5839Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
5840 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5841 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
5842 }
5843 return Register();
5844}
5845
5846Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
5847 if ((Subtarget->hasFP16())) {
5848 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
5849 }
5850 return Register();
5851}
5852
5853Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
5854switch (RetVT.SimpleTy) {
5855 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
5856 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
5857 default: return Register();
5858}
5859}
5860
5861Register fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
5862 if (RetVT.SimpleTy != MVT::v32i16)
5863 return Register();
5864 if ((Subtarget->hasFP16())) {
5865 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
5866 }
5867 return Register();
5868}
5869
5870Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
5871 if ((Subtarget->hasVLX())) {
5872 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5873 }
5874 return Register();
5875}
5876
5877Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
5878 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5879 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5880 }
5881 return Register();
5882}
5883
5884Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
5885 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5886 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5887 }
5888 return Register();
5889}
5890
5891Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
5892switch (RetVT.SimpleTy) {
5893 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
5894 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
5895 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
5896 default: return Register();
5897}
5898}
5899
5900Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
5901 if ((Subtarget->hasVLX())) {
5902 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
5903 }
5904 return Register();
5905}
5906
5907Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
5908 if ((Subtarget->hasDQI())) {
5909 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
5910 }
5911 return Register();
5912}
5913
5914Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
5915switch (RetVT.SimpleTy) {
5916 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
5917 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
5918 default: return Register();
5919}
5920}
5921
5922Register fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
5923 if (RetVT.SimpleTy != MVT::v16i32)
5924 return Register();
5925 if ((Subtarget->hasAVX512())) {
5926 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
5927 }
5928 return Register();
5929}
5930
5931Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
5932 if ((Subtarget->hasVLX())) {
5933 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
5934 }
5935 return Register();
5936}
5937
5938Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
5939 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5940 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
5941 }
5942 return Register();
5943}
5944
5945Register fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
5946switch (RetVT.SimpleTy) {
5947 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
5948 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
5949 default: return Register();
5950}
5951}
5952
5953Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
5954 if ((Subtarget->hasVLX())) {
5955 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
5956 }
5957 return Register();
5958}
5959
5960Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
5961 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5962 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
5963 }
5964 return Register();
5965}
5966
5967Register fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
5968switch (RetVT.SimpleTy) {
5969 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
5970 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
5971 default: return Register();
5972}
5973}
5974
5975Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
5976 if ((Subtarget->hasAVX512())) {
5977 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
5978 }
5979 return Register();
5980}
5981
5982Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
5983 if ((Subtarget->hasDQI())) {
5984 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
5985 }
5986 return Register();
5987}
5988
5989Register fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
5990switch (RetVT.SimpleTy) {
5991 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
5992 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
5993 default: return Register();
5994}
5995}
5996
5997Register fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
5998 switch (VT.SimpleTy) {
5999 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
6000 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
6001 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
6002 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
6003 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
6004 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
6005 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
6006 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
6007 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
6008 default: return Register();
6009 }
6010}
6011
6012// FastEmit functions for X86ISD::CVTTP2UIS.
6013
6014Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i32_r(Register Op0) {
6015 if ((Subtarget->hasAVX10_2())) {
6016 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6017 }
6018 return Register();
6019}
6020
6021Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v2i64_r(Register Op0) {
6022 if ((Subtarget->hasAVX10_2())) {
6023 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6024 }
6025 return Register();
6026}
6027
6028Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i64_r(Register Op0) {
6029 if ((Subtarget->hasAVX10_2())) {
6030 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ256rr, RC: &X86::VR256XRegClass, Op0);
6031 }
6032 return Register();
6033}
6034
6035Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6036switch (RetVT.SimpleTy) {
6037 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i32_r(Op0);
6038 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v2i64_r(Op0);
6039 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_MVT_v4i64_r(Op0);
6040 default: return Register();
6041}
6042}
6043
6044Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i32_r(Register Op0) {
6045 if ((Subtarget->hasAVX10_2())) {
6046 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZ256rr, RC: &X86::VR256XRegClass, Op0);
6047 }
6048 return Register();
6049}
6050
6051Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i64_r(Register Op0) {
6052 if ((Subtarget->hasAVX10_2())) {
6053 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZrr, RC: &X86::VR512RegClass, Op0);
6054 }
6055 return Register();
6056}
6057
6058Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_r(MVT RetVT, Register Op0) {
6059switch (RetVT.SimpleTy) {
6060 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i32_r(Op0);
6061 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_MVT_v8i64_r(Op0);
6062 default: return Register();
6063}
6064}
6065
6066Register fastEmit_X86ISD_CVTTP2UIS_MVT_v16f32_r(MVT RetVT, Register Op0) {
6067 if (RetVT.SimpleTy != MVT::v16i32)
6068 return Register();
6069 if ((Subtarget->hasAVX10_2())) {
6070 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZrr, RC: &X86::VR512RegClass, Op0);
6071 }
6072 return Register();
6073}
6074
6075Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v4i32_r(Register Op0) {
6076 if ((Subtarget->hasAVX10_2())) {
6077 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6078 }
6079 return Register();
6080}
6081
6082Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v2i64_r(Register Op0) {
6083 if ((Subtarget->hasAVX10_2())) {
6084 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
6085 }
6086 return Register();
6087}
6088
6089Register fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6090switch (RetVT.SimpleTy) {
6091 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v4i32_r(Op0);
6092 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_MVT_v2i64_r(Op0);
6093 default: return Register();
6094}
6095}
6096
6097Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i32_r(Register Op0) {
6098 if ((Subtarget->hasAVX10_2())) {
6099 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ256rr, RC: &X86::VR128XRegClass, Op0);
6100 }
6101 return Register();
6102}
6103
6104Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i64_r(Register Op0) {
6105 if ((Subtarget->hasAVX10_2())) {
6106 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ256rr, RC: &X86::VR256XRegClass, Op0);
6107 }
6108 return Register();
6109}
6110
6111Register fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_r(MVT RetVT, Register Op0) {
6112switch (RetVT.SimpleTy) {
6113 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i32_r(Op0);
6114 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_MVT_v4i64_r(Op0);
6115 default: return Register();
6116}
6117}
6118
6119Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6120 if ((Subtarget->hasAVX10_2())) {
6121 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZrr, RC: &X86::VR256XRegClass, Op0);
6122 }
6123 return Register();
6124}
6125
6126Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6127 if ((Subtarget->hasAVX10_2())) {
6128 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZrr, RC: &X86::VR512RegClass, Op0);
6129 }
6130 return Register();
6131}
6132
6133Register fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_r(MVT RetVT, Register Op0) {
6134switch (RetVT.SimpleTy) {
6135 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i32_r(Op0);
6136 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_MVT_v8i64_r(Op0);
6137 default: return Register();
6138}
6139}
6140
6141Register fastEmit_X86ISD_CVTTP2UIS_r(MVT VT, MVT RetVT, Register Op0) {
6142 switch (VT.SimpleTy) {
6143 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f32_r(RetVT, Op0);
6144 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f32_r(RetVT, Op0);
6145 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UIS_MVT_v16f32_r(RetVT, Op0);
6146 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v2f64_r(RetVT, Op0);
6147 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v4f64_r(RetVT, Op0);
6148 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UIS_MVT_v8f64_r(RetVT, Op0);
6149 default: return Register();
6150 }
6151}
6152
6153// FastEmit functions for X86ISD::CVTTP2UIS_SAE.
6154
6155Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6156 if (RetVT.SimpleTy != MVT::v4i64)
6157 return Register();
6158 if ((Subtarget->hasAVX10_2())) {
6159 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
6160 }
6161 return Register();
6162}
6163
6164Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
6165 if (RetVT.SimpleTy != MVT::v8i64)
6166 return Register();
6167 if ((Subtarget->hasAVX10_2())) {
6168 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZrrb, RC: &X86::VR512RegClass, Op0);
6169 }
6170 return Register();
6171}
6172
6173Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6174 if (RetVT.SimpleTy != MVT::v16i32)
6175 return Register();
6176 if ((Subtarget->hasAVX10_2())) {
6177 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQSZrrb, RC: &X86::VR512RegClass, Op0);
6178 }
6179 return Register();
6180}
6181
6182Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i32_r(Register Op0) {
6183 if ((Subtarget->hasAVX10_2())) {
6184 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ256rrb, RC: &X86::VR128XRegClass, Op0);
6185 }
6186 return Register();
6187}
6188
6189Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i64_r(Register Op0) {
6190 if ((Subtarget->hasAVX10_2())) {
6191 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZ256rrb, RC: &X86::VR256XRegClass, Op0);
6192 }
6193 return Register();
6194}
6195
6196Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_r(MVT RetVT, Register Op0) {
6197switch (RetVT.SimpleTy) {
6198 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i32_r(Op0);
6199 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_MVT_v4i64_r(Op0);
6200 default: return Register();
6201}
6202}
6203
6204Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6205 if ((Subtarget->hasAVX10_2())) {
6206 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZrrb, RC: &X86::VR256XRegClass, Op0);
6207 }
6208 return Register();
6209}
6210
6211Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6212 if ((Subtarget->hasAVX10_2())) {
6213 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQSZrrb, RC: &X86::VR512RegClass, Op0);
6214 }
6215 return Register();
6216}
6217
6218Register fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6219switch (RetVT.SimpleTy) {
6220 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
6221 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
6222 default: return Register();
6223}
6224}
6225
6226Register fastEmit_X86ISD_CVTTP2UIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6227 switch (VT.SimpleTy) {
6228 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f32_r(RetVT, Op0);
6229 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f32_r(RetVT, Op0);
6230 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v16f32_r(RetVT, Op0);
6231 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v4f64_r(RetVT, Op0);
6232 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UIS_SAE_MVT_v8f64_r(RetVT, Op0);
6233 default: return Register();
6234 }
6235}
6236
6237// FastEmit functions for X86ISD::CVTTP2UI_SAE.
6238
6239Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6240 if (RetVT.SimpleTy != MVT::v8i64)
6241 return Register();
6242 if ((Subtarget->hasFP16())) {
6243 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6244 }
6245 return Register();
6246}
6247
6248Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
6249 if (RetVT.SimpleTy != MVT::v16i32)
6250 return Register();
6251 if ((Subtarget->hasFP16())) {
6252 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrrb, RC: &X86::VR512RegClass, Op0);
6253 }
6254 return Register();
6255}
6256
6257Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
6258 if (RetVT.SimpleTy != MVT::v32i16)
6259 return Register();
6260 if ((Subtarget->hasFP16())) {
6261 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrrb, RC: &X86::VR512RegClass, Op0);
6262 }
6263 return Register();
6264}
6265
6266Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
6267 if (RetVT.SimpleTy != MVT::v8i64)
6268 return Register();
6269 if ((Subtarget->hasDQI())) {
6270 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6271 }
6272 return Register();
6273}
6274
6275Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
6276 if (RetVT.SimpleTy != MVT::v16i32)
6277 return Register();
6278 if ((Subtarget->hasAVX512())) {
6279 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrrb, RC: &X86::VR512RegClass, Op0);
6280 }
6281 return Register();
6282}
6283
6284Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Register Op0) {
6285 if ((Subtarget->hasAVX512())) {
6286 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrrb, RC: &X86::VR256XRegClass, Op0);
6287 }
6288 return Register();
6289}
6290
6291Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Register Op0) {
6292 if ((Subtarget->hasDQI())) {
6293 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrrb, RC: &X86::VR512RegClass, Op0);
6294 }
6295 return Register();
6296}
6297
6298Register fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
6299switch (RetVT.SimpleTy) {
6300 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
6301 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
6302 default: return Register();
6303}
6304}
6305
6306Register fastEmit_X86ISD_CVTTP2UI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6307 switch (VT.SimpleTy) {
6308 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(RetVT, Op0);
6309 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(RetVT, Op0);
6310 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(RetVT, Op0);
6311 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(RetVT, Op0);
6312 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(RetVT, Op0);
6313 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(RetVT, Op0);
6314 default: return Register();
6315 }
6316}
6317
6318// FastEmit functions for X86ISD::CVTTS2SI.
6319
6320Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Register Op0) {
6321 if ((Subtarget->hasFP16())) {
6322 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6323 }
6324 return Register();
6325}
6326
6327Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Register Op0) {
6328 if ((Subtarget->hasFP16())) {
6329 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6330 }
6331 return Register();
6332}
6333
6334Register fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
6335switch (RetVT.SimpleTy) {
6336 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Op0);
6337 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Op0);
6338 default: return Register();
6339}
6340}
6341
6342Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Register Op0) {
6343 if ((Subtarget->hasAVX512())) {
6344 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6345 }
6346 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6347 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6348 }
6349 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6350 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6351 }
6352 return Register();
6353}
6354
6355Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Register Op0) {
6356 if ((Subtarget->hasAVX512())) {
6357 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6358 }
6359 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6360 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6361 }
6362 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6363 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6364 }
6365 return Register();
6366}
6367
6368Register fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
6369switch (RetVT.SimpleTy) {
6370 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0);
6371 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0);
6372 default: return Register();
6373}
6374}
6375
6376Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Register Op0) {
6377 if ((Subtarget->hasAVX512())) {
6378 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrr_Int, RC: &X86::GR32RegClass, Op0);
6379 }
6380 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6381 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6382 }
6383 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6384 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIrr_Int, RC: &X86::GR32RegClass, Op0);
6385 }
6386 return Register();
6387}
6388
6389Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Register Op0) {
6390 if ((Subtarget->hasAVX512())) {
6391 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6392 }
6393 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6394 return fastEmitInst_r(MachineInstOpcode: X86::CVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6395 }
6396 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6397 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64rr_Int, RC: &X86::GR64RegClass, Op0);
6398 }
6399 return Register();
6400}
6401
6402Register fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
6403switch (RetVT.SimpleTy) {
6404 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0);
6405 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0);
6406 default: return Register();
6407}
6408}
6409
6410Register fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, Register Op0) {
6411 switch (VT.SimpleTy) {
6412 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(RetVT, Op0);
6413 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0);
6414 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0);
6415 default: return Register();
6416 }
6417}
6418
6419// FastEmit functions for X86ISD::CVTTS2SIS.
6420
6421Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i32_r(Register Op0) {
6422 if ((Subtarget->hasAVX10_2())) {
6423 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SISrr_Int, RC: &X86::GR32RegClass, Op0);
6424 }
6425 return Register();
6426}
6427
6428Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i64_r(Register Op0) {
6429 if ((Subtarget->hasAVX10_2())) {
6430 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6431 }
6432 return Register();
6433}
6434
6435Register fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6436switch (RetVT.SimpleTy) {
6437 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i32_r(Op0);
6438 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_MVT_i64_r(Op0);
6439 default: return Register();
6440}
6441}
6442
6443Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i32_r(Register Op0) {
6444 if ((Subtarget->hasAVX10_2())) {
6445 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SISrr_Int, RC: &X86::GR32RegClass, Op0);
6446 }
6447 return Register();
6448}
6449
6450Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i64_r(Register Op0) {
6451 if ((Subtarget->hasAVX10_2())) {
6452 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6453 }
6454 return Register();
6455}
6456
6457Register fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6458switch (RetVT.SimpleTy) {
6459 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i32_r(Op0);
6460 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_MVT_i64_r(Op0);
6461 default: return Register();
6462}
6463}
6464
6465Register fastEmit_X86ISD_CVTTS2SIS_r(MVT VT, MVT RetVT, Register Op0) {
6466 switch (VT.SimpleTy) {
6467 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SIS_MVT_v4f32_r(RetVT, Op0);
6468 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SIS_MVT_v2f64_r(RetVT, Op0);
6469 default: return Register();
6470 }
6471}
6472
6473// FastEmit functions for X86ISD::CVTTS2SIS_SAE.
6474
6475Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6476 if ((Subtarget->hasAVX10_2())) {
6477 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SISrrb_Int, RC: &X86::GR32RegClass, Op0);
6478 }
6479 return Register();
6480}
6481
6482Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6483 if ((Subtarget->hasAVX10_2())) {
6484 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6485 }
6486 return Register();
6487}
6488
6489Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6490switch (RetVT.SimpleTy) {
6491 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i32_r(Op0);
6492 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_MVT_i64_r(Op0);
6493 default: return Register();
6494}
6495}
6496
6497Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6498 if ((Subtarget->hasAVX10_2())) {
6499 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SISrrb_Int, RC: &X86::GR32RegClass, Op0);
6500 }
6501 return Register();
6502}
6503
6504Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6505 if ((Subtarget->hasAVX10_2())) {
6506 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6507 }
6508 return Register();
6509}
6510
6511Register fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6512switch (RetVT.SimpleTy) {
6513 case MVT::i32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i32_r(Op0);
6514 case MVT::i64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_MVT_i64_r(Op0);
6515 default: return Register();
6516}
6517}
6518
6519Register fastEmit_X86ISD_CVTTS2SIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6520 switch (VT.SimpleTy) {
6521 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v4f32_r(RetVT, Op0);
6522 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SIS_SAE_MVT_v2f64_r(RetVT, Op0);
6523 default: return Register();
6524 }
6525}
6526
6527// FastEmit functions for X86ISD::CVTTS2SI_SAE.
6528
6529Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Register Op0) {
6530 if ((Subtarget->hasFP16())) {
6531 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6532 }
6533 return Register();
6534}
6535
6536Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Register Op0) {
6537 if ((Subtarget->hasFP16())) {
6538 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6539 }
6540 return Register();
6541}
6542
6543Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6544switch (RetVT.SimpleTy) {
6545 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Op0);
6546 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Op0);
6547 default: return Register();
6548}
6549}
6550
6551Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6552 if ((Subtarget->hasAVX512())) {
6553 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6554 }
6555 return Register();
6556}
6557
6558Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6559 if ((Subtarget->hasAVX512())) {
6560 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6561 }
6562 return Register();
6563}
6564
6565Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6566switch (RetVT.SimpleTy) {
6567 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Op0);
6568 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Op0);
6569 default: return Register();
6570}
6571}
6572
6573Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6574 if ((Subtarget->hasAVX512())) {
6575 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6576 }
6577 return Register();
6578}
6579
6580Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6581 if ((Subtarget->hasAVX512())) {
6582 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2SI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6583 }
6584 return Register();
6585}
6586
6587Register fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6588switch (RetVT.SimpleTy) {
6589 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Op0);
6590 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Op0);
6591 default: return Register();
6592}
6593}
6594
6595Register fastEmit_X86ISD_CVTTS2SI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6596 switch (VT.SimpleTy) {
6597 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(RetVT, Op0);
6598 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(RetVT, Op0);
6599 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(RetVT, Op0);
6600 default: return Register();
6601 }
6602}
6603
6604// FastEmit functions for X86ISD::CVTTS2UI.
6605
6606Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Register Op0) {
6607 if ((Subtarget->hasFP16())) {
6608 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6609 }
6610 return Register();
6611}
6612
6613Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Register Op0) {
6614 if ((Subtarget->hasFP16())) {
6615 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6616 }
6617 return Register();
6618}
6619
6620Register fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
6621switch (RetVT.SimpleTy) {
6622 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Op0);
6623 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Op0);
6624 default: return Register();
6625}
6626}
6627
6628Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Register Op0) {
6629 if ((Subtarget->hasAVX512())) {
6630 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6631 }
6632 return Register();
6633}
6634
6635Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Register Op0) {
6636 if ((Subtarget->hasAVX512())) {
6637 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6638 }
6639 return Register();
6640}
6641
6642Register fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
6643switch (RetVT.SimpleTy) {
6644 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0);
6645 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0);
6646 default: return Register();
6647}
6648}
6649
6650Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Register Op0) {
6651 if ((Subtarget->hasAVX512())) {
6652 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrr_Int, RC: &X86::GR32RegClass, Op0);
6653 }
6654 return Register();
6655}
6656
6657Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Register Op0) {
6658 if ((Subtarget->hasAVX512())) {
6659 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrr_Int, RC: &X86::GR64RegClass, Op0);
6660 }
6661 return Register();
6662}
6663
6664Register fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
6665switch (RetVT.SimpleTy) {
6666 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0);
6667 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0);
6668 default: return Register();
6669}
6670}
6671
6672Register fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, Register Op0) {
6673 switch (VT.SimpleTy) {
6674 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(RetVT, Op0);
6675 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0);
6676 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0);
6677 default: return Register();
6678 }
6679}
6680
6681// FastEmit functions for X86ISD::CVTTS2UIS.
6682
6683Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i32_r(Register Op0) {
6684 if ((Subtarget->hasAVX10_2())) {
6685 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USISrr_Int, RC: &X86::GR32RegClass, Op0);
6686 }
6687 return Register();
6688}
6689
6690Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i64_r(Register Op0) {
6691 if ((Subtarget->hasAVX10_2())) {
6692 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6693 }
6694 return Register();
6695}
6696
6697Register fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_r(MVT RetVT, Register Op0) {
6698switch (RetVT.SimpleTy) {
6699 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i32_r(Op0);
6700 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_MVT_i64_r(Op0);
6701 default: return Register();
6702}
6703}
6704
6705Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i32_r(Register Op0) {
6706 if ((Subtarget->hasAVX10_2())) {
6707 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USISrr_Int, RC: &X86::GR32RegClass, Op0);
6708 }
6709 return Register();
6710}
6711
6712Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i64_r(Register Op0) {
6713 if ((Subtarget->hasAVX10_2())) {
6714 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Srr_Int, RC: &X86::GR64RegClass, Op0);
6715 }
6716 return Register();
6717}
6718
6719Register fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_r(MVT RetVT, Register Op0) {
6720switch (RetVT.SimpleTy) {
6721 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i32_r(Op0);
6722 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_MVT_i64_r(Op0);
6723 default: return Register();
6724}
6725}
6726
6727Register fastEmit_X86ISD_CVTTS2UIS_r(MVT VT, MVT RetVT, Register Op0) {
6728 switch (VT.SimpleTy) {
6729 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UIS_MVT_v4f32_r(RetVT, Op0);
6730 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UIS_MVT_v2f64_r(RetVT, Op0);
6731 default: return Register();
6732 }
6733}
6734
6735// FastEmit functions for X86ISD::CVTTS2UIS_SAE.
6736
6737Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6738 if ((Subtarget->hasAVX10_2())) {
6739 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USISrrb_Int, RC: &X86::GR32RegClass, Op0);
6740 }
6741 return Register();
6742}
6743
6744Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6745 if ((Subtarget->hasAVX10_2())) {
6746 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6747 }
6748 return Register();
6749}
6750
6751Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6752switch (RetVT.SimpleTy) {
6753 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i32_r(Op0);
6754 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_MVT_i64_r(Op0);
6755 default: return Register();
6756}
6757}
6758
6759Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6760 if ((Subtarget->hasAVX10_2())) {
6761 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USISrrb_Int, RC: &X86::GR32RegClass, Op0);
6762 }
6763 return Register();
6764}
6765
6766Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6767 if ((Subtarget->hasAVX10_2())) {
6768 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Srrb_Int, RC: &X86::GR64RegClass, Op0);
6769 }
6770 return Register();
6771}
6772
6773Register fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6774switch (RetVT.SimpleTy) {
6775 case MVT::i32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i32_r(Op0);
6776 case MVT::i64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_MVT_i64_r(Op0);
6777 default: return Register();
6778}
6779}
6780
6781Register fastEmit_X86ISD_CVTTS2UIS_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6782 switch (VT.SimpleTy) {
6783 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v4f32_r(RetVT, Op0);
6784 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UIS_SAE_MVT_v2f64_r(RetVT, Op0);
6785 default: return Register();
6786 }
6787}
6788
6789// FastEmit functions for X86ISD::CVTTS2UI_SAE.
6790
6791Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Register Op0) {
6792 if ((Subtarget->hasFP16())) {
6793 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6794 }
6795 return Register();
6796}
6797
6798Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Register Op0) {
6799 if ((Subtarget->hasFP16())) {
6800 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSH2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6801 }
6802 return Register();
6803}
6804
6805Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
6806switch (RetVT.SimpleTy) {
6807 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Op0);
6808 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Op0);
6809 default: return Register();
6810}
6811}
6812
6813Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Register Op0) {
6814 if ((Subtarget->hasAVX512())) {
6815 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6816 }
6817 return Register();
6818}
6819
6820Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Register Op0) {
6821 if ((Subtarget->hasAVX512())) {
6822 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSS2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6823 }
6824 return Register();
6825}
6826
6827Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(MVT RetVT, Register Op0) {
6828switch (RetVT.SimpleTy) {
6829 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Op0);
6830 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Op0);
6831 default: return Register();
6832}
6833}
6834
6835Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Register Op0) {
6836 if ((Subtarget->hasAVX512())) {
6837 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USIZrrb_Int, RC: &X86::GR32RegClass, Op0);
6838 }
6839 return Register();
6840}
6841
6842Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Register Op0) {
6843 if ((Subtarget->hasAVX512())) {
6844 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTSD2USI64Zrrb_Int, RC: &X86::GR64RegClass, Op0);
6845 }
6846 return Register();
6847}
6848
6849Register fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(MVT RetVT, Register Op0) {
6850switch (RetVT.SimpleTy) {
6851 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Op0);
6852 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Op0);
6853 default: return Register();
6854}
6855}
6856
6857Register fastEmit_X86ISD_CVTTS2UI_SAE_r(MVT VT, MVT RetVT, Register Op0) {
6858 switch (VT.SimpleTy) {
6859 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(RetVT, Op0);
6860 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(RetVT, Op0);
6861 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(RetVT, Op0);
6862 default: return Register();
6863 }
6864}
6865
6866// FastEmit functions for X86ISD::CVTUI2P.
6867
6868Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
6869 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6870 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6871 }
6872 return Register();
6873}
6874
6875Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
6876 if ((Subtarget->hasVLX())) {
6877 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
6878 }
6879 return Register();
6880}
6881
6882Register fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
6883switch (RetVT.SimpleTy) {
6884 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
6885 case MVT::v2f64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
6886 default: return Register();
6887}
6888}
6889
6890Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
6891 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6892 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
6893 }
6894 return Register();
6895}
6896
6897Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
6898 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6899 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
6900 }
6901 return Register();
6902}
6903
6904Register fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
6905switch (RetVT.SimpleTy) {
6906 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
6907 case MVT::v4f32: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
6908 default: return Register();
6909}
6910}
6911
6912Register fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
6913 if (RetVT.SimpleTy != MVT::v8f16)
6914 return Register();
6915 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6916 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
6917 }
6918 return Register();
6919}
6920
6921Register fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, Register Op0) {
6922 switch (VT.SimpleTy) {
6923 case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
6924 case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
6925 case MVT::v4i64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
6926 default: return Register();
6927 }
6928}
6929
6930// FastEmit functions for X86ISD::DYN_ALLOCA.
6931
6932Register fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
6933 if (RetVT.SimpleTy != MVT::isVoid)
6934 return Register();
6935 if ((!Subtarget->isTarget64BitLP64())) {
6936 return fastEmitInst_r(MachineInstOpcode: X86::DYN_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
6937 }
6938 return Register();
6939}
6940
6941Register fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
6942 if (RetVT.SimpleTy != MVT::isVoid)
6943 return Register();
6944 if ((Subtarget->is64Bit())) {
6945 return fastEmitInst_r(MachineInstOpcode: X86::DYN_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
6946 }
6947 return Register();
6948}
6949
6950Register fastEmit_X86ISD_DYN_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
6951 switch (VT.SimpleTy) {
6952 case MVT::i32: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(RetVT, Op0);
6953 case MVT::i64: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(RetVT, Op0);
6954 default: return Register();
6955 }
6956}
6957
6958// FastEmit functions for X86ISD::EH_RETURN.
6959
6960Register fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, Register Op0) {
6961 if (RetVT.SimpleTy != MVT::isVoid)
6962 return Register();
6963 return fastEmitInst_r(MachineInstOpcode: X86::EH_RETURN, RC: &X86::GR32RegClass, Op0);
6964}
6965
6966Register fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, Register Op0) {
6967 if (RetVT.SimpleTy != MVT::isVoid)
6968 return Register();
6969 return fastEmitInst_r(MachineInstOpcode: X86::EH_RETURN64, RC: &X86::GR64RegClass, Op0);
6970}
6971
6972Register fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, Register Op0) {
6973 switch (VT.SimpleTy) {
6974 case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0);
6975 case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0);
6976 default: return Register();
6977 }
6978}
6979
6980// FastEmit functions for X86ISD::FGETEXP.
6981
6982Register fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(MVT RetVT, Register Op0) {
6983 if (RetVT.SimpleTy != MVT::v8f16)
6984 return Register();
6985 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6986 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ128r, RC: &X86::VR128XRegClass, Op0);
6987 }
6988 return Register();
6989}
6990
6991Register fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(MVT RetVT, Register Op0) {
6992 if (RetVT.SimpleTy != MVT::v16f16)
6993 return Register();
6994 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6995 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZ256r, RC: &X86::VR256XRegClass, Op0);
6996 }
6997 return Register();
6998}
6999
7000Register fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(MVT RetVT, Register Op0) {
7001 if (RetVT.SimpleTy != MVT::v32f16)
7002 return Register();
7003 if ((Subtarget->hasFP16())) {
7004 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZr, RC: &X86::VR512RegClass, Op0);
7005 }
7006 return Register();
7007}
7008
7009Register fastEmit_X86ISD_FGETEXP_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7010 if (RetVT.SimpleTy != MVT::v8bf16)
7011 return Register();
7012 if ((Subtarget->hasAVX10_2())) {
7013 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7014 }
7015 return Register();
7016}
7017
7018Register fastEmit_X86ISD_FGETEXP_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7019 if (RetVT.SimpleTy != MVT::v16bf16)
7020 return Register();
7021 if ((Subtarget->hasAVX10_2())) {
7022 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7023 }
7024 return Register();
7025}
7026
7027Register fastEmit_X86ISD_FGETEXP_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7028 if (RetVT.SimpleTy != MVT::v32bf16)
7029 return Register();
7030 if ((Subtarget->hasAVX10_2())) {
7031 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPBF16Zr, RC: &X86::VR512RegClass, Op0);
7032 }
7033 return Register();
7034}
7035
7036Register fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7037 if (RetVT.SimpleTy != MVT::v4f32)
7038 return Register();
7039 if ((Subtarget->hasVLX())) {
7040 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ128r, RC: &X86::VR128XRegClass, Op0);
7041 }
7042 return Register();
7043}
7044
7045Register fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7046 if (RetVT.SimpleTy != MVT::v8f32)
7047 return Register();
7048 if ((Subtarget->hasVLX())) {
7049 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZ256r, RC: &X86::VR256XRegClass, Op0);
7050 }
7051 return Register();
7052}
7053
7054Register fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7055 if (RetVT.SimpleTy != MVT::v16f32)
7056 return Register();
7057 if ((Subtarget->hasAVX512())) {
7058 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZr, RC: &X86::VR512RegClass, Op0);
7059 }
7060 return Register();
7061}
7062
7063Register fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(MVT RetVT, Register Op0) {
7064 if (RetVT.SimpleTy != MVT::v2f64)
7065 return Register();
7066 if ((Subtarget->hasVLX())) {
7067 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ128r, RC: &X86::VR128XRegClass, Op0);
7068 }
7069 return Register();
7070}
7071
7072Register fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(MVT RetVT, Register Op0) {
7073 if (RetVT.SimpleTy != MVT::v4f64)
7074 return Register();
7075 if ((Subtarget->hasVLX())) {
7076 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZ256r, RC: &X86::VR256XRegClass, Op0);
7077 }
7078 return Register();
7079}
7080
7081Register fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(MVT RetVT, Register Op0) {
7082 if (RetVT.SimpleTy != MVT::v8f64)
7083 return Register();
7084 if ((Subtarget->hasAVX512())) {
7085 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZr, RC: &X86::VR512RegClass, Op0);
7086 }
7087 return Register();
7088}
7089
7090Register fastEmit_X86ISD_FGETEXP_r(MVT VT, MVT RetVT, Register Op0) {
7091 switch (VT.SimpleTy) {
7092 case MVT::v8f16: return fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(RetVT, Op0);
7093 case MVT::v16f16: return fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(RetVT, Op0);
7094 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(RetVT, Op0);
7095 case MVT::v8bf16: return fastEmit_X86ISD_FGETEXP_MVT_v8bf16_r(RetVT, Op0);
7096 case MVT::v16bf16: return fastEmit_X86ISD_FGETEXP_MVT_v16bf16_r(RetVT, Op0);
7097 case MVT::v32bf16: return fastEmit_X86ISD_FGETEXP_MVT_v32bf16_r(RetVT, Op0);
7098 case MVT::v4f32: return fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(RetVT, Op0);
7099 case MVT::v8f32: return fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(RetVT, Op0);
7100 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(RetVT, Op0);
7101 case MVT::v2f64: return fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(RetVT, Op0);
7102 case MVT::v4f64: return fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(RetVT, Op0);
7103 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(RetVT, Op0);
7104 default: return Register();
7105 }
7106}
7107
7108// FastEmit functions for X86ISD::FGETEXP_SAE.
7109
7110Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(MVT RetVT, Register Op0) {
7111 if (RetVT.SimpleTy != MVT::v32f16)
7112 return Register();
7113 if ((Subtarget->hasFP16())) {
7114 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPHZrb, RC: &X86::VR512RegClass, Op0);
7115 }
7116 return Register();
7117}
7118
7119Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(MVT RetVT, Register Op0) {
7120 if (RetVT.SimpleTy != MVT::v16f32)
7121 return Register();
7122 if ((Subtarget->hasAVX512())) {
7123 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPSZrb, RC: &X86::VR512RegClass, Op0);
7124 }
7125 return Register();
7126}
7127
7128Register fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(MVT RetVT, Register Op0) {
7129 if (RetVT.SimpleTy != MVT::v8f64)
7130 return Register();
7131 if ((Subtarget->hasAVX512())) {
7132 return fastEmitInst_r(MachineInstOpcode: X86::VGETEXPPDZrb, RC: &X86::VR512RegClass, Op0);
7133 }
7134 return Register();
7135}
7136
7137Register fastEmit_X86ISD_FGETEXP_SAE_r(MVT VT, MVT RetVT, Register Op0) {
7138 switch (VT.SimpleTy) {
7139 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(RetVT, Op0);
7140 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(RetVT, Op0);
7141 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(RetVT, Op0);
7142 default: return Register();
7143 }
7144}
7145
7146// FastEmit functions for X86ISD::FP_TO_SINT_SAT.
7147
7148Register fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7149 if (RetVT.SimpleTy != MVT::v2i64)
7150 return Register();
7151 if ((Subtarget->hasAVX10_2())) {
7152 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7153 }
7154 return Register();
7155}
7156
7157Register fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
7158 if (RetVT.SimpleTy != MVT::v4i32)
7159 return Register();
7160 if ((Subtarget->hasAVX10_2())) {
7161 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7162 }
7163 return Register();
7164}
7165
7166Register fastEmit_X86ISD_FP_TO_SINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
7167 switch (VT.SimpleTy) {
7168 case MVT::v4f32: return fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v4f32_r(RetVT, Op0);
7169 case MVT::v2f64: return fastEmit_X86ISD_FP_TO_SINT_SAT_MVT_v2f64_r(RetVT, Op0);
7170 default: return Register();
7171 }
7172}
7173
7174// FastEmit functions for X86ISD::FP_TO_UINT_SAT.
7175
7176Register fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7177 if (RetVT.SimpleTy != MVT::v2i64)
7178 return Register();
7179 if ((Subtarget->hasAVX10_2())) {
7180 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7181 }
7182 return Register();
7183}
7184
7185Register fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
7186 if (RetVT.SimpleTy != MVT::v4i32)
7187 return Register();
7188 if ((Subtarget->hasAVX10_2())) {
7189 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQSZ128rr, RC: &X86::VR128XRegClass, Op0);
7190 }
7191 return Register();
7192}
7193
7194Register fastEmit_X86ISD_FP_TO_UINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
7195 switch (VT.SimpleTy) {
7196 case MVT::v4f32: return fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v4f32_r(RetVT, Op0);
7197 case MVT::v2f64: return fastEmit_X86ISD_FP_TO_UINT_SAT_MVT_v2f64_r(RetVT, Op0);
7198 default: return Register();
7199 }
7200}
7201
7202// FastEmit functions for X86ISD::FRCP.
7203
7204Register fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, Register Op0) {
7205 if (RetVT.SimpleTy != MVT::f32)
7206 return Register();
7207 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7208 return fastEmitInst_r(MachineInstOpcode: X86::RCPSSr, RC: &X86::FR32RegClass, Op0);
7209 }
7210 return Register();
7211}
7212
7213Register fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7214 if (RetVT.SimpleTy != MVT::v4f32)
7215 return Register();
7216 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7217 return fastEmitInst_r(MachineInstOpcode: X86::RCPPSr, RC: &X86::VR128RegClass, Op0);
7218 }
7219 if ((Subtarget->hasAVX())) {
7220 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSr, RC: &X86::VR128RegClass, Op0);
7221 }
7222 return Register();
7223}
7224
7225Register fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7226 if (RetVT.SimpleTy != MVT::v8f32)
7227 return Register();
7228 if ((Subtarget->hasAVX())) {
7229 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPSYr, RC: &X86::VR256RegClass, Op0);
7230 }
7231 return Register();
7232}
7233
7234Register fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, Register Op0) {
7235 switch (VT.SimpleTy) {
7236 case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0);
7237 case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0);
7238 case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0);
7239 default: return Register();
7240 }
7241}
7242
7243// FastEmit functions for X86ISD::FRSQRT.
7244
7245Register fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
7246 if (RetVT.SimpleTy != MVT::f32)
7247 return Register();
7248 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7249 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTSSr, RC: &X86::FR32RegClass, Op0);
7250 }
7251 return Register();
7252}
7253
7254Register fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
7255 if (RetVT.SimpleTy != MVT::v4f32)
7256 return Register();
7257 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7258 return fastEmitInst_r(MachineInstOpcode: X86::RSQRTPSr, RC: &X86::VR128RegClass, Op0);
7259 }
7260 if ((Subtarget->hasAVX())) {
7261 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSr, RC: &X86::VR128RegClass, Op0);
7262 }
7263 return Register();
7264}
7265
7266Register fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, Register Op0) {
7267 if (RetVT.SimpleTy != MVT::v8f32)
7268 return Register();
7269 if ((Subtarget->hasAVX())) {
7270 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPSYr, RC: &X86::VR256RegClass, Op0);
7271 }
7272 return Register();
7273}
7274
7275Register fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, Register Op0) {
7276 switch (VT.SimpleTy) {
7277 case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0);
7278 case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0);
7279 case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0);
7280 default: return Register();
7281 }
7282}
7283
7284// FastEmit functions for X86ISD::MMX_MOVD2W.
7285
7286Register fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, Register Op0) {
7287 if (RetVT.SimpleTy != MVT::i32)
7288 return Register();
7289 if ((Subtarget->hasMMX())) {
7290 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64grr, RC: &X86::GR32RegClass, Op0);
7291 }
7292 return Register();
7293}
7294
7295Register fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, Register Op0) {
7296 switch (VT.SimpleTy) {
7297 case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0);
7298 default: return Register();
7299 }
7300}
7301
7302// FastEmit functions for X86ISD::MMX_MOVW2D.
7303
7304Register fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, Register Op0) {
7305 if (RetVT.SimpleTy != MVT::x86mmx)
7306 return Register();
7307 if ((Subtarget->hasMMX())) {
7308 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVD64rr, RC: &X86::VR64RegClass, Op0);
7309 }
7310 return Register();
7311}
7312
7313Register fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, Register Op0) {
7314 switch (VT.SimpleTy) {
7315 case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0);
7316 default: return Register();
7317 }
7318}
7319
7320// FastEmit functions for X86ISD::MOVDDUP.
7321
7322Register fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, Register Op0) {
7323 if (RetVT.SimpleTy != MVT::v2f64)
7324 return Register();
7325 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7326 return fastEmitInst_r(MachineInstOpcode: X86::MOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7327 }
7328 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7329 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
7330 }
7331 return Register();
7332}
7333
7334Register fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, Register Op0) {
7335 if (RetVT.SimpleTy != MVT::v4f64)
7336 return Register();
7337 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7338 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7339 }
7340 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7341 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPYrr, RC: &X86::VR256RegClass, Op0);
7342 }
7343 return Register();
7344}
7345
7346Register fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, Register Op0) {
7347 if (RetVT.SimpleTy != MVT::v8f64)
7348 return Register();
7349 if ((Subtarget->hasAVX512())) {
7350 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZrr, RC: &X86::VR512RegClass, Op0);
7351 }
7352 return Register();
7353}
7354
7355Register fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, Register Op0) {
7356 switch (VT.SimpleTy) {
7357 case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0);
7358 case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0);
7359 case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0);
7360 default: return Register();
7361 }
7362}
7363
7364// FastEmit functions for X86ISD::MOVDQ2Q.
7365
7366Register fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, Register Op0) {
7367 if (RetVT.SimpleTy != MVT::x86mmx)
7368 return Register();
7369 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
7370 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVDQ2Qrr, RC: &X86::VR64RegClass, Op0);
7371 }
7372 return Register();
7373}
7374
7375Register fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, Register Op0) {
7376 switch (VT.SimpleTy) {
7377 case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0);
7378 default: return Register();
7379 }
7380}
7381
7382// FastEmit functions for X86ISD::MOVMSK.
7383
7384Register fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, Register Op0) {
7385 if (RetVT.SimpleTy != MVT::i32)
7386 return Register();
7387 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7388 return fastEmitInst_r(MachineInstOpcode: X86::PMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
7389 }
7390 if ((Subtarget->hasAVX())) {
7391 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBrr, RC: &X86::GR32RegClass, Op0);
7392 }
7393 return Register();
7394}
7395
7396Register fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, Register Op0) {
7397 if (RetVT.SimpleTy != MVT::i32)
7398 return Register();
7399 if ((Subtarget->hasAVX2())) {
7400 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVMSKBYrr, RC: &X86::GR32RegClass, Op0);
7401 }
7402 return Register();
7403}
7404
7405Register fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, Register Op0) {
7406 if (RetVT.SimpleTy != MVT::i32)
7407 return Register();
7408 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7409 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7410 }
7411 if ((Subtarget->hasAVX())) {
7412 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7413 }
7414 return Register();
7415}
7416
7417Register fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, Register Op0) {
7418 if (RetVT.SimpleTy != MVT::i32)
7419 return Register();
7420 if ((Subtarget->hasAVX())) {
7421 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
7422 }
7423 return Register();
7424}
7425
7426Register fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, Register Op0) {
7427 if (RetVT.SimpleTy != MVT::i32)
7428 return Register();
7429 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7430 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7431 }
7432 if ((Subtarget->hasAVX())) {
7433 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7434 }
7435 return Register();
7436}
7437
7438Register fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, Register Op0) {
7439 if (RetVT.SimpleTy != MVT::i32)
7440 return Register();
7441 if ((Subtarget->hasAVX())) {
7442 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
7443 }
7444 return Register();
7445}
7446
7447Register fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, Register Op0) {
7448 if (RetVT.SimpleTy != MVT::i32)
7449 return Register();
7450 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7451 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7452 }
7453 if ((Subtarget->hasAVX())) {
7454 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSrr, RC: &X86::GR32RegClass, Op0);
7455 }
7456 return Register();
7457}
7458
7459Register fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, Register Op0) {
7460 if (RetVT.SimpleTy != MVT::i32)
7461 return Register();
7462 if ((Subtarget->hasAVX())) {
7463 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPSYrr, RC: &X86::GR32RegClass, Op0);
7464 }
7465 return Register();
7466}
7467
7468Register fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, Register Op0) {
7469 if (RetVT.SimpleTy != MVT::i32)
7470 return Register();
7471 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7472 return fastEmitInst_r(MachineInstOpcode: X86::MOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7473 }
7474 if ((Subtarget->hasAVX())) {
7475 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDrr, RC: &X86::GR32RegClass, Op0);
7476 }
7477 return Register();
7478}
7479
7480Register fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, Register Op0) {
7481 if (RetVT.SimpleTy != MVT::i32)
7482 return Register();
7483 if ((Subtarget->hasAVX())) {
7484 return fastEmitInst_r(MachineInstOpcode: X86::VMOVMSKPDYrr, RC: &X86::GR32RegClass, Op0);
7485 }
7486 return Register();
7487}
7488
7489Register fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, Register Op0) {
7490 switch (VT.SimpleTy) {
7491 case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0);
7492 case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0);
7493 case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0);
7494 case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0);
7495 case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0);
7496 case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0);
7497 case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0);
7498 case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0);
7499 case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0);
7500 case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0);
7501 default: return Register();
7502 }
7503}
7504
7505// FastEmit functions for X86ISD::MOVQ2DQ.
7506
7507Register fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(MVT RetVT, Register Op0) {
7508 if (RetVT.SimpleTy != MVT::v2i64)
7509 return Register();
7510 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
7511 return fastEmitInst_r(MachineInstOpcode: X86::MMX_MOVQ2DQrr, RC: &X86::VR128RegClass, Op0);
7512 }
7513 return Register();
7514}
7515
7516Register fastEmit_X86ISD_MOVQ2DQ_r(MVT VT, MVT RetVT, Register Op0) {
7517 switch (VT.SimpleTy) {
7518 case MVT::x86mmx: return fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(RetVT, Op0);
7519 default: return Register();
7520 }
7521}
7522
7523// FastEmit functions for X86ISD::MOVSHDUP.
7524
7525Register fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, Register Op0) {
7526 if (RetVT.SimpleTy != MVT::v4i32)
7527 return Register();
7528 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7529 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7530 }
7531 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7532 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7533 }
7534 return Register();
7535}
7536
7537Register fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, Register Op0) {
7538 if (RetVT.SimpleTy != MVT::v8i32)
7539 return Register();
7540 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7541 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
7542 }
7543 return Register();
7544}
7545
7546Register fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7547 if (RetVT.SimpleTy != MVT::v4f32)
7548 return Register();
7549 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7550 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7551 }
7552 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7553 return fastEmitInst_r(MachineInstOpcode: X86::MOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7554 }
7555 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7556 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPrr, RC: &X86::VR128RegClass, Op0);
7557 }
7558 return Register();
7559}
7560
7561Register fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7562 if (RetVT.SimpleTy != MVT::v8f32)
7563 return Register();
7564 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7565 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7566 }
7567 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7568 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPYrr, RC: &X86::VR256RegClass, Op0);
7569 }
7570 return Register();
7571}
7572
7573Register fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7574 if (RetVT.SimpleTy != MVT::v16f32)
7575 return Register();
7576 if ((Subtarget->hasAVX512())) {
7577 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSHDUPZrr, RC: &X86::VR512RegClass, Op0);
7578 }
7579 return Register();
7580}
7581
7582Register fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, Register Op0) {
7583 switch (VT.SimpleTy) {
7584 case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0);
7585 case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0);
7586 case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0);
7587 case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0);
7588 case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0);
7589 default: return Register();
7590 }
7591}
7592
7593// FastEmit functions for X86ISD::MOVSLDUP.
7594
7595Register fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, Register Op0) {
7596 if (RetVT.SimpleTy != MVT::v4i32)
7597 return Register();
7598 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7599 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7600 }
7601 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7602 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7603 }
7604 return Register();
7605}
7606
7607Register fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, Register Op0) {
7608 if (RetVT.SimpleTy != MVT::v8i32)
7609 return Register();
7610 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7611 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
7612 }
7613 return Register();
7614}
7615
7616Register fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, Register Op0) {
7617 if (RetVT.SimpleTy != MVT::v4f32)
7618 return Register();
7619 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7620 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
7621 }
7622 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
7623 return fastEmitInst_r(MachineInstOpcode: X86::MOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7624 }
7625 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7626 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPrr, RC: &X86::VR128RegClass, Op0);
7627 }
7628 return Register();
7629}
7630
7631Register fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, Register Op0) {
7632 if (RetVT.SimpleTy != MVT::v8f32)
7633 return Register();
7634 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7635 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZ256rr, RC: &X86::VR256XRegClass, Op0);
7636 }
7637 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7638 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPYrr, RC: &X86::VR256RegClass, Op0);
7639 }
7640 return Register();
7641}
7642
7643Register fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, Register Op0) {
7644 if (RetVT.SimpleTy != MVT::v16f32)
7645 return Register();
7646 if ((Subtarget->hasAVX512())) {
7647 return fastEmitInst_r(MachineInstOpcode: X86::VMOVSLDUPZrr, RC: &X86::VR512RegClass, Op0);
7648 }
7649 return Register();
7650}
7651
7652Register fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, Register Op0) {
7653 switch (VT.SimpleTy) {
7654 case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0);
7655 case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0);
7656 case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0);
7657 case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0);
7658 case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0);
7659 default: return Register();
7660 }
7661}
7662
7663// FastEmit functions for X86ISD::NT_BRIND.
7664
7665Register fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, Register Op0) {
7666 if (RetVT.SimpleTy != MVT::isVoid)
7667 return Register();
7668 if ((!Subtarget->is64Bit())) {
7669 return fastEmitInst_r(MachineInstOpcode: X86::JMP16r_NT, RC: &X86::GR16RegClass, Op0);
7670 }
7671 return Register();
7672}
7673
7674Register fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
7675 if (RetVT.SimpleTy != MVT::isVoid)
7676 return Register();
7677 if ((!Subtarget->is64Bit())) {
7678 return fastEmitInst_r(MachineInstOpcode: X86::JMP32r_NT, RC: &X86::GR32RegClass, Op0);
7679 }
7680 return Register();
7681}
7682
7683Register fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
7684 if (RetVT.SimpleTy != MVT::isVoid)
7685 return Register();
7686 if ((Subtarget->is64Bit())) {
7687 return fastEmitInst_r(MachineInstOpcode: X86::JMP64r_NT, RC: &X86::GR64RegClass, Op0);
7688 }
7689 return Register();
7690}
7691
7692Register fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
7693 switch (VT.SimpleTy) {
7694 case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0);
7695 case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0);
7696 case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0);
7697 default: return Register();
7698 }
7699}
7700
7701// FastEmit functions for X86ISD::NT_CALL.
7702
7703Register fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, Register Op0) {
7704 if (RetVT.SimpleTy != MVT::isVoid)
7705 return Register();
7706 if ((!Subtarget->is64Bit())) {
7707 return fastEmitInst_r(MachineInstOpcode: X86::CALL16r_NT, RC: &X86::GR16RegClass, Op0);
7708 }
7709 return Register();
7710}
7711
7712Register fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
7713 if (RetVT.SimpleTy != MVT::isVoid)
7714 return Register();
7715 if ((!Subtarget->is64Bit())) {
7716 return fastEmitInst_r(MachineInstOpcode: X86::CALL32r_NT, RC: &X86::GR32RegClass, Op0);
7717 }
7718 return Register();
7719}
7720
7721Register fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
7722 if (RetVT.SimpleTy != MVT::isVoid)
7723 return Register();
7724 if ((Subtarget->is64Bit())) {
7725 return fastEmitInst_r(MachineInstOpcode: X86::CALL64r_NT, RC: &X86::GR64RegClass, Op0);
7726 }
7727 return Register();
7728}
7729
7730Register fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, Register Op0) {
7731 switch (VT.SimpleTy) {
7732 case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0);
7733 case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0);
7734 case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0);
7735 default: return Register();
7736 }
7737}
7738
7739// FastEmit functions for X86ISD::PHMINPOS.
7740
7741Register fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, Register Op0) {
7742 if (RetVT.SimpleTy != MVT::v8i16)
7743 return Register();
7744 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
7745 return fastEmitInst_r(MachineInstOpcode: X86::PHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
7746 }
7747 if ((Subtarget->hasAVX())) {
7748 return fastEmitInst_r(MachineInstOpcode: X86::VPHMINPOSUWrr, RC: &X86::VR128RegClass, Op0);
7749 }
7750 return Register();
7751}
7752
7753Register fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, Register Op0) {
7754 switch (VT.SimpleTy) {
7755 case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0);
7756 default: return Register();
7757 }
7758}
7759
7760// FastEmit functions for X86ISD::PROBED_ALLOCA.
7761
7762Register fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
7763 if (RetVT.SimpleTy != MVT::i32)
7764 return Register();
7765 if ((!Subtarget->isTarget64BitLP64())) {
7766 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
7767 }
7768 return Register();
7769}
7770
7771Register fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
7772 if (RetVT.SimpleTy != MVT::i64)
7773 return Register();
7774 if ((Subtarget->is64Bit())) {
7775 return fastEmitInst_r(MachineInstOpcode: X86::PROBED_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
7776 }
7777 return Register();
7778}
7779
7780Register fastEmit_X86ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
7781 switch (VT.SimpleTy) {
7782 case MVT::i32: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(RetVT, Op0);
7783 case MVT::i64: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
7784 default: return Register();
7785 }
7786}
7787
7788// FastEmit functions for X86ISD::RCP14.
7789
7790Register fastEmit_X86ISD_RCP14_MVT_v8f16_r(MVT RetVT, Register Op0) {
7791 if (RetVT.SimpleTy != MVT::v8f16)
7792 return Register();
7793 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7794 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ128r, RC: &X86::VR128XRegClass, Op0);
7795 }
7796 return Register();
7797}
7798
7799Register fastEmit_X86ISD_RCP14_MVT_v16f16_r(MVT RetVT, Register Op0) {
7800 if (RetVT.SimpleTy != MVT::v16f16)
7801 return Register();
7802 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7803 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZ256r, RC: &X86::VR256XRegClass, Op0);
7804 }
7805 return Register();
7806}
7807
7808Register fastEmit_X86ISD_RCP14_MVT_v32f16_r(MVT RetVT, Register Op0) {
7809 if (RetVT.SimpleTy != MVT::v32f16)
7810 return Register();
7811 if ((Subtarget->hasFP16())) {
7812 return fastEmitInst_r(MachineInstOpcode: X86::VRCPPHZr, RC: &X86::VR512RegClass, Op0);
7813 }
7814 return Register();
7815}
7816
7817Register fastEmit_X86ISD_RCP14_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7818 if (RetVT.SimpleTy != MVT::v8bf16)
7819 return Register();
7820 if ((Subtarget->hasAVX10_2())) {
7821 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7822 }
7823 return Register();
7824}
7825
7826Register fastEmit_X86ISD_RCP14_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7827 if (RetVT.SimpleTy != MVT::v16bf16)
7828 return Register();
7829 if ((Subtarget->hasAVX10_2())) {
7830 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7831 }
7832 return Register();
7833}
7834
7835Register fastEmit_X86ISD_RCP14_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7836 if (RetVT.SimpleTy != MVT::v32bf16)
7837 return Register();
7838 if ((Subtarget->hasAVX10_2())) {
7839 return fastEmitInst_r(MachineInstOpcode: X86::VRCPBF16Zr, RC: &X86::VR512RegClass, Op0);
7840 }
7841 return Register();
7842}
7843
7844Register fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, Register Op0) {
7845 if (RetVT.SimpleTy != MVT::v4f32)
7846 return Register();
7847 if ((Subtarget->hasVLX())) {
7848 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ128r, RC: &X86::VR128XRegClass, Op0);
7849 }
7850 return Register();
7851}
7852
7853Register fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, Register Op0) {
7854 if (RetVT.SimpleTy != MVT::v8f32)
7855 return Register();
7856 if ((Subtarget->hasVLX())) {
7857 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZ256r, RC: &X86::VR256XRegClass, Op0);
7858 }
7859 return Register();
7860}
7861
7862Register fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, Register Op0) {
7863 if (RetVT.SimpleTy != MVT::v16f32)
7864 return Register();
7865 if ((Subtarget->hasAVX512())) {
7866 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PSZr, RC: &X86::VR512RegClass, Op0);
7867 }
7868 return Register();
7869}
7870
7871Register fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, Register Op0) {
7872 if (RetVT.SimpleTy != MVT::v2f64)
7873 return Register();
7874 if ((Subtarget->hasVLX())) {
7875 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ128r, RC: &X86::VR128XRegClass, Op0);
7876 }
7877 return Register();
7878}
7879
7880Register fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, Register Op0) {
7881 if (RetVT.SimpleTy != MVT::v4f64)
7882 return Register();
7883 if ((Subtarget->hasVLX())) {
7884 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZ256r, RC: &X86::VR256XRegClass, Op0);
7885 }
7886 return Register();
7887}
7888
7889Register fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, Register Op0) {
7890 if (RetVT.SimpleTy != MVT::v8f64)
7891 return Register();
7892 if ((Subtarget->hasAVX512())) {
7893 return fastEmitInst_r(MachineInstOpcode: X86::VRCP14PDZr, RC: &X86::VR512RegClass, Op0);
7894 }
7895 return Register();
7896}
7897
7898Register fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, Register Op0) {
7899 switch (VT.SimpleTy) {
7900 case MVT::v8f16: return fastEmit_X86ISD_RCP14_MVT_v8f16_r(RetVT, Op0);
7901 case MVT::v16f16: return fastEmit_X86ISD_RCP14_MVT_v16f16_r(RetVT, Op0);
7902 case MVT::v32f16: return fastEmit_X86ISD_RCP14_MVT_v32f16_r(RetVT, Op0);
7903 case MVT::v8bf16: return fastEmit_X86ISD_RCP14_MVT_v8bf16_r(RetVT, Op0);
7904 case MVT::v16bf16: return fastEmit_X86ISD_RCP14_MVT_v16bf16_r(RetVT, Op0);
7905 case MVT::v32bf16: return fastEmit_X86ISD_RCP14_MVT_v32bf16_r(RetVT, Op0);
7906 case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0);
7907 case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0);
7908 case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0);
7909 case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0);
7910 case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0);
7911 case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0);
7912 default: return Register();
7913 }
7914}
7915
7916// FastEmit functions for X86ISD::RSQRT14.
7917
7918Register fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(MVT RetVT, Register Op0) {
7919 if (RetVT.SimpleTy != MVT::v8f16)
7920 return Register();
7921 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7922 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ128r, RC: &X86::VR128XRegClass, Op0);
7923 }
7924 return Register();
7925}
7926
7927Register fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(MVT RetVT, Register Op0) {
7928 if (RetVT.SimpleTy != MVT::v16f16)
7929 return Register();
7930 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7931 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZ256r, RC: &X86::VR256XRegClass, Op0);
7932 }
7933 return Register();
7934}
7935
7936Register fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(MVT RetVT, Register Op0) {
7937 if (RetVT.SimpleTy != MVT::v32f16)
7938 return Register();
7939 if ((Subtarget->hasFP16())) {
7940 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTPHZr, RC: &X86::VR512RegClass, Op0);
7941 }
7942 return Register();
7943}
7944
7945Register fastEmit_X86ISD_RSQRT14_MVT_v8bf16_r(MVT RetVT, Register Op0) {
7946 if (RetVT.SimpleTy != MVT::v8bf16)
7947 return Register();
7948 if ((Subtarget->hasAVX10_2())) {
7949 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Z128r, RC: &X86::VR128XRegClass, Op0);
7950 }
7951 return Register();
7952}
7953
7954Register fastEmit_X86ISD_RSQRT14_MVT_v16bf16_r(MVT RetVT, Register Op0) {
7955 if (RetVT.SimpleTy != MVT::v16bf16)
7956 return Register();
7957 if ((Subtarget->hasAVX10_2())) {
7958 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Z256r, RC: &X86::VR256XRegClass, Op0);
7959 }
7960 return Register();
7961}
7962
7963Register fastEmit_X86ISD_RSQRT14_MVT_v32bf16_r(MVT RetVT, Register Op0) {
7964 if (RetVT.SimpleTy != MVT::v32bf16)
7965 return Register();
7966 if ((Subtarget->hasAVX10_2())) {
7967 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRTBF16Zr, RC: &X86::VR512RegClass, Op0);
7968 }
7969 return Register();
7970}
7971
7972Register fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, Register Op0) {
7973 if (RetVT.SimpleTy != MVT::v4f32)
7974 return Register();
7975 if ((Subtarget->hasVLX())) {
7976 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ128r, RC: &X86::VR128XRegClass, Op0);
7977 }
7978 return Register();
7979}
7980
7981Register fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, Register Op0) {
7982 if (RetVT.SimpleTy != MVT::v8f32)
7983 return Register();
7984 if ((Subtarget->hasVLX())) {
7985 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZ256r, RC: &X86::VR256XRegClass, Op0);
7986 }
7987 return Register();
7988}
7989
7990Register fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, Register Op0) {
7991 if (RetVT.SimpleTy != MVT::v16f32)
7992 return Register();
7993 if ((Subtarget->hasAVX512())) {
7994 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PSZr, RC: &X86::VR512RegClass, Op0);
7995 }
7996 return Register();
7997}
7998
7999Register fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, Register Op0) {
8000 if (RetVT.SimpleTy != MVT::v2f64)
8001 return Register();
8002 if ((Subtarget->hasVLX())) {
8003 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ128r, RC: &X86::VR128XRegClass, Op0);
8004 }
8005 return Register();
8006}
8007
8008Register fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, Register Op0) {
8009 if (RetVT.SimpleTy != MVT::v4f64)
8010 return Register();
8011 if ((Subtarget->hasVLX())) {
8012 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZ256r, RC: &X86::VR256XRegClass, Op0);
8013 }
8014 return Register();
8015}
8016
8017Register fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, Register Op0) {
8018 if (RetVT.SimpleTy != MVT::v8f64)
8019 return Register();
8020 if ((Subtarget->hasAVX512())) {
8021 return fastEmitInst_r(MachineInstOpcode: X86::VRSQRT14PDZr, RC: &X86::VR512RegClass, Op0);
8022 }
8023 return Register();
8024}
8025
8026Register fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, Register Op0) {
8027 switch (VT.SimpleTy) {
8028 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(RetVT, Op0);
8029 case MVT::v16f16: return fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(RetVT, Op0);
8030 case MVT::v32f16: return fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(RetVT, Op0);
8031 case MVT::v8bf16: return fastEmit_X86ISD_RSQRT14_MVT_v8bf16_r(RetVT, Op0);
8032 case MVT::v16bf16: return fastEmit_X86ISD_RSQRT14_MVT_v16bf16_r(RetVT, Op0);
8033 case MVT::v32bf16: return fastEmit_X86ISD_RSQRT14_MVT_v32bf16_r(RetVT, Op0);
8034 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0);
8035 case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0);
8036 case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0);
8037 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0);
8038 case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0);
8039 case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0);
8040 default: return Register();
8041 }
8042}
8043
8044// FastEmit functions for X86ISD::SEG_ALLOCA.
8045
8046Register fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, Register Op0) {
8047 if (RetVT.SimpleTy != MVT::i32)
8048 return Register();
8049 if ((!Subtarget->isTarget64BitLP64())) {
8050 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_32, RC: &X86::GR32RegClass, Op0);
8051 }
8052 return Register();
8053}
8054
8055Register fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
8056 if (RetVT.SimpleTy != MVT::i64)
8057 return Register();
8058 if ((Subtarget->is64Bit())) {
8059 return fastEmitInst_r(MachineInstOpcode: X86::SEG_ALLOCA_64, RC: &X86::GR64RegClass, Op0);
8060 }
8061 return Register();
8062}
8063
8064Register fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
8065 switch (VT.SimpleTy) {
8066 case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0);
8067 case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0);
8068 default: return Register();
8069 }
8070}
8071
8072// FastEmit functions for X86ISD::STRICT_CVTPH2PS.
8073
8074Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Register Op0) {
8075 if ((Subtarget->hasVLX())) {
8076 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8077 }
8078 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
8079 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSrr, RC: &X86::VR128RegClass, Op0);
8080 }
8081 return Register();
8082}
8083
8084Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Register Op0) {
8085 if ((Subtarget->hasVLX())) {
8086 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZ256rr, RC: &X86::VR256XRegClass, Op0);
8087 }
8088 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
8089 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSYrr, RC: &X86::VR256RegClass, Op0);
8090 }
8091 return Register();
8092}
8093
8094Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(MVT RetVT, Register Op0) {
8095switch (RetVT.SimpleTy) {
8096 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
8097 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
8098 default: return Register();
8099}
8100}
8101
8102Register fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(MVT RetVT, Register Op0) {
8103 if (RetVT.SimpleTy != MVT::v16f32)
8104 return Register();
8105 if ((Subtarget->hasAVX512())) {
8106 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSZrr, RC: &X86::VR512RegClass, Op0);
8107 }
8108 return Register();
8109}
8110
8111Register fastEmit_X86ISD_STRICT_CVTPH2PS_r(MVT VT, MVT RetVT, Register Op0) {
8112 switch (VT.SimpleTy) {
8113 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
8114 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
8115 default: return Register();
8116 }
8117}
8118
8119// FastEmit functions for X86ISD::STRICT_CVTSI2P.
8120
8121Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
8122 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8123 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8124 }
8125 return Register();
8126}
8127
8128Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
8129 if ((Subtarget->hasVLX())) {
8130 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8131 }
8132 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8133 return fastEmitInst_r(MachineInstOpcode: X86::CVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
8134 }
8135 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8136 return fastEmitInst_r(MachineInstOpcode: X86::VCVTDQ2PDrr, RC: &X86::VR128RegClass, Op0);
8137 }
8138 return Register();
8139}
8140
8141Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
8142switch (RetVT.SimpleTy) {
8143 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
8144 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
8145 default: return Register();
8146}
8147}
8148
8149Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
8150 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8151 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8152 }
8153 return Register();
8154}
8155
8156Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
8157 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8158 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8159 }
8160 return Register();
8161}
8162
8163Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
8164switch (RetVT.SimpleTy) {
8165 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
8166 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
8167 default: return Register();
8168}
8169}
8170
8171Register fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
8172 if (RetVT.SimpleTy != MVT::v8f16)
8173 return Register();
8174 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8175 return fastEmitInst_r(MachineInstOpcode: X86::VCVTQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8176 }
8177 return Register();
8178}
8179
8180Register fastEmit_X86ISD_STRICT_CVTSI2P_r(MVT VT, MVT RetVT, Register Op0) {
8181 switch (VT.SimpleTy) {
8182 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
8183 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
8184 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
8185 default: return Register();
8186 }
8187}
8188
8189// FastEmit functions for X86ISD::STRICT_CVTTP2SI.
8190
8191Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
8192 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8193 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ128rr, RC: &X86::VR128XRegClass, Op0);
8194 }
8195 return Register();
8196}
8197
8198Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
8199 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8200 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8201 }
8202 return Register();
8203}
8204
8205Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
8206 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8207 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
8208 }
8209 return Register();
8210}
8211
8212Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
8213 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8214 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8215 }
8216 return Register();
8217}
8218
8219Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
8220 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8221 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8222 }
8223 return Register();
8224}
8225
8226Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
8227 if ((Subtarget->hasFP16())) {
8228 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2QQZrr, RC: &X86::VR512RegClass, Op0);
8229 }
8230 return Register();
8231}
8232
8233Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(MVT RetVT, Register Op0) {
8234switch (RetVT.SimpleTy) {
8235 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
8236 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
8237 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
8238 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
8239 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
8240 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
8241 default: return Register();
8242}
8243}
8244
8245Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
8246 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8247 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZ256rr, RC: &X86::VR256XRegClass, Op0);
8248 }
8249 return Register();
8250}
8251
8252Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
8253 if ((Subtarget->hasFP16())) {
8254 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2DQZrr, RC: &X86::VR512RegClass, Op0);
8255 }
8256 return Register();
8257}
8258
8259Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(MVT RetVT, Register Op0) {
8260switch (RetVT.SimpleTy) {
8261 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
8262 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
8263 default: return Register();
8264}
8265}
8266
8267Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(MVT RetVT, Register Op0) {
8268 if (RetVT.SimpleTy != MVT::v32i16)
8269 return Register();
8270 if ((Subtarget->hasFP16())) {
8271 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2WZrr, RC: &X86::VR512RegClass, Op0);
8272 }
8273 return Register();
8274}
8275
8276Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
8277 if ((Subtarget->hasVLX())) {
8278 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8279 }
8280 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8281 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
8282 }
8283 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8284 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQrr, RC: &X86::VR128RegClass, Op0);
8285 }
8286 return Register();
8287}
8288
8289Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
8290 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8291 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8292 }
8293 return Register();
8294}
8295
8296Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
8297 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8298 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8299 }
8300 return Register();
8301}
8302
8303Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(MVT RetVT, Register Op0) {
8304switch (RetVT.SimpleTy) {
8305 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
8306 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
8307 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
8308 default: return Register();
8309}
8310}
8311
8312Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
8313 if ((Subtarget->hasVLX())) {
8314 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZ256rr, RC: &X86::VR256XRegClass, Op0);
8315 }
8316 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8317 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQYrr, RC: &X86::VR256RegClass, Op0);
8318 }
8319 return Register();
8320}
8321
8322Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
8323 if ((Subtarget->hasDQI())) {
8324 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2QQZrr, RC: &X86::VR512RegClass, Op0);
8325 }
8326 return Register();
8327}
8328
8329Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(MVT RetVT, Register Op0) {
8330switch (RetVT.SimpleTy) {
8331 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
8332 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
8333 default: return Register();
8334}
8335}
8336
8337Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(MVT RetVT, Register Op0) {
8338 if (RetVT.SimpleTy != MVT::v16i32)
8339 return Register();
8340 if ((Subtarget->hasAVX512())) {
8341 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2DQZrr, RC: &X86::VR512RegClass, Op0);
8342 }
8343 return Register();
8344}
8345
8346Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
8347 if ((Subtarget->hasVLX())) {
8348 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ128rr, RC: &X86::VR128XRegClass, Op0);
8349 }
8350 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8351 return fastEmitInst_r(MachineInstOpcode: X86::CVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
8352 }
8353 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8354 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQrr, RC: &X86::VR128RegClass, Op0);
8355 }
8356 return Register();
8357}
8358
8359Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
8360 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8361 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ128rr, RC: &X86::VR128XRegClass, Op0);
8362 }
8363 return Register();
8364}
8365
8366Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(MVT RetVT, Register Op0) {
8367switch (RetVT.SimpleTy) {
8368 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
8369 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
8370 default: return Register();
8371}
8372}
8373
8374Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
8375 if ((Subtarget->hasVLX())) {
8376 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZ256rr, RC: &X86::VR128XRegClass, Op0);
8377 }
8378 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8379 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQYrr, RC: &X86::VR128RegClass, Op0);
8380 }
8381 return Register();
8382}
8383
8384Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
8385 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8386 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZ256rr, RC: &X86::VR256XRegClass, Op0);
8387 }
8388 return Register();
8389}
8390
8391Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(MVT RetVT, Register Op0) {
8392switch (RetVT.SimpleTy) {
8393 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
8394 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
8395 default: return Register();
8396}
8397}
8398
8399Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
8400 if ((Subtarget->hasAVX512())) {
8401 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2DQZrr, RC: &X86::VR256XRegClass, Op0);
8402 }
8403 return Register();
8404}
8405
8406Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
8407 if ((Subtarget->hasDQI())) {
8408 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2QQZrr, RC: &X86::VR512RegClass, Op0);
8409 }
8410 return Register();
8411}
8412
8413Register fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(MVT RetVT, Register Op0) {
8414switch (RetVT.SimpleTy) {
8415 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
8416 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
8417 default: return Register();
8418}
8419}
8420
8421Register fastEmit_X86ISD_STRICT_CVTTP2SI_r(MVT VT, MVT RetVT, Register Op0) {
8422 switch (VT.SimpleTy) {
8423 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
8424 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
8425 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
8426 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
8427 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
8428 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
8429 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
8430 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
8431 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
8432 default: return Register();
8433 }
8434}
8435
8436// FastEmit functions for X86ISD::STRICT_CVTTP2UI.
8437
8438Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Register Op0) {
8439 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8440 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ128rr, RC: &X86::VR128XRegClass, Op0);
8441 }
8442 return Register();
8443}
8444
8445Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Register Op0) {
8446 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8447 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8448 }
8449 return Register();
8450}
8451
8452Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Register Op0) {
8453 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8454 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
8455 }
8456 return Register();
8457}
8458
8459Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Register Op0) {
8460 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8461 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8462 }
8463 return Register();
8464}
8465
8466Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Register Op0) {
8467 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8468 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8469 }
8470 return Register();
8471}
8472
8473Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Register Op0) {
8474 if ((Subtarget->hasFP16())) {
8475 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UQQZrr, RC: &X86::VR512RegClass, Op0);
8476 }
8477 return Register();
8478}
8479
8480Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(MVT RetVT, Register Op0) {
8481switch (RetVT.SimpleTy) {
8482 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
8483 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
8484 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
8485 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
8486 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
8487 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
8488 default: return Register();
8489}
8490}
8491
8492Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Register Op0) {
8493 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8494 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZ256rr, RC: &X86::VR256XRegClass, Op0);
8495 }
8496 return Register();
8497}
8498
8499Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Register Op0) {
8500 if ((Subtarget->hasFP16())) {
8501 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UDQZrr, RC: &X86::VR512RegClass, Op0);
8502 }
8503 return Register();
8504}
8505
8506Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(MVT RetVT, Register Op0) {
8507switch (RetVT.SimpleTy) {
8508 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
8509 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
8510 default: return Register();
8511}
8512}
8513
8514Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(MVT RetVT, Register Op0) {
8515 if (RetVT.SimpleTy != MVT::v32i16)
8516 return Register();
8517 if ((Subtarget->hasFP16())) {
8518 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPH2UWZrr, RC: &X86::VR512RegClass, Op0);
8519 }
8520 return Register();
8521}
8522
8523Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Register Op0) {
8524 if ((Subtarget->hasVLX())) {
8525 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8526 }
8527 return Register();
8528}
8529
8530Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Register Op0) {
8531 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8532 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8533 }
8534 return Register();
8535}
8536
8537Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Register Op0) {
8538 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8539 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8540 }
8541 return Register();
8542}
8543
8544Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(MVT RetVT, Register Op0) {
8545switch (RetVT.SimpleTy) {
8546 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
8547 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
8548 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
8549 default: return Register();
8550}
8551}
8552
8553Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Register Op0) {
8554 if ((Subtarget->hasVLX())) {
8555 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZ256rr, RC: &X86::VR256XRegClass, Op0);
8556 }
8557 return Register();
8558}
8559
8560Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Register Op0) {
8561 if ((Subtarget->hasDQI())) {
8562 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UQQZrr, RC: &X86::VR512RegClass, Op0);
8563 }
8564 return Register();
8565}
8566
8567Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(MVT RetVT, Register Op0) {
8568switch (RetVT.SimpleTy) {
8569 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
8570 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
8571 default: return Register();
8572}
8573}
8574
8575Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(MVT RetVT, Register Op0) {
8576 if (RetVT.SimpleTy != MVT::v16i32)
8577 return Register();
8578 if ((Subtarget->hasAVX512())) {
8579 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPS2UDQZrr, RC: &X86::VR512RegClass, Op0);
8580 }
8581 return Register();
8582}
8583
8584Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Register Op0) {
8585 if ((Subtarget->hasVLX())) {
8586 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ128rr, RC: &X86::VR128XRegClass, Op0);
8587 }
8588 return Register();
8589}
8590
8591Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Register Op0) {
8592 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8593 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ128rr, RC: &X86::VR128XRegClass, Op0);
8594 }
8595 return Register();
8596}
8597
8598Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(MVT RetVT, Register Op0) {
8599switch (RetVT.SimpleTy) {
8600 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
8601 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
8602 default: return Register();
8603}
8604}
8605
8606Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Register Op0) {
8607 if ((Subtarget->hasVLX())) {
8608 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZ256rr, RC: &X86::VR128XRegClass, Op0);
8609 }
8610 return Register();
8611}
8612
8613Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Register Op0) {
8614 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8615 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZ256rr, RC: &X86::VR256XRegClass, Op0);
8616 }
8617 return Register();
8618}
8619
8620Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(MVT RetVT, Register Op0) {
8621switch (RetVT.SimpleTy) {
8622 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
8623 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
8624 default: return Register();
8625}
8626}
8627
8628Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Register Op0) {
8629 if ((Subtarget->hasAVX512())) {
8630 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UDQZrr, RC: &X86::VR256XRegClass, Op0);
8631 }
8632 return Register();
8633}
8634
8635Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Register Op0) {
8636 if ((Subtarget->hasDQI())) {
8637 return fastEmitInst_r(MachineInstOpcode: X86::VCVTTPD2UQQZrr, RC: &X86::VR512RegClass, Op0);
8638 }
8639 return Register();
8640}
8641
8642Register fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(MVT RetVT, Register Op0) {
8643switch (RetVT.SimpleTy) {
8644 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
8645 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
8646 default: return Register();
8647}
8648}
8649
8650Register fastEmit_X86ISD_STRICT_CVTTP2UI_r(MVT VT, MVT RetVT, Register Op0) {
8651 switch (VT.SimpleTy) {
8652 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
8653 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
8654 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
8655 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
8656 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
8657 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
8658 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
8659 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
8660 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
8661 default: return Register();
8662 }
8663}
8664
8665// FastEmit functions for X86ISD::STRICT_CVTUI2P.
8666
8667Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Register Op0) {
8668 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8669 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8670 }
8671 return Register();
8672}
8673
8674Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Register Op0) {
8675 if ((Subtarget->hasVLX())) {
8676 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUDQ2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8677 }
8678 return Register();
8679}
8680
8681Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(MVT RetVT, Register Op0) {
8682switch (RetVT.SimpleTy) {
8683 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
8684 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
8685 default: return Register();
8686}
8687}
8688
8689Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Register Op0) {
8690 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8691 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8692 }
8693 return Register();
8694}
8695
8696Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Register Op0) {
8697 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8698 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8699 }
8700 return Register();
8701}
8702
8703Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(MVT RetVT, Register Op0) {
8704switch (RetVT.SimpleTy) {
8705 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
8706 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
8707 default: return Register();
8708}
8709}
8710
8711Register fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(MVT RetVT, Register Op0) {
8712 if (RetVT.SimpleTy != MVT::v8f16)
8713 return Register();
8714 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8715 return fastEmitInst_r(MachineInstOpcode: X86::VCVTUQQ2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8716 }
8717 return Register();
8718}
8719
8720Register fastEmit_X86ISD_STRICT_CVTUI2P_r(MVT VT, MVT RetVT, Register Op0) {
8721 switch (VT.SimpleTy) {
8722 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
8723 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
8724 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
8725 default: return Register();
8726 }
8727}
8728
8729// FastEmit functions for X86ISD::STRICT_VFPEXT.
8730
8731Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Register Op0) {
8732 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8733 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
8734 }
8735 return Register();
8736}
8737
8738Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Register Op0) {
8739 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8740 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8741 }
8742 return Register();
8743}
8744
8745Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Register Op0) {
8746 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8747 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
8748 }
8749 return Register();
8750}
8751
8752Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(MVT RetVT, Register Op0) {
8753switch (RetVT.SimpleTy) {
8754 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
8755 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
8756 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
8757 default: return Register();
8758}
8759}
8760
8761Register fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(MVT RetVT, Register Op0) {
8762 if (RetVT.SimpleTy != MVT::v2f64)
8763 return Register();
8764 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8765 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
8766 }
8767 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8768 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
8769 }
8770 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8771 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
8772 }
8773 return Register();
8774}
8775
8776Register fastEmit_X86ISD_STRICT_VFPEXT_r(MVT VT, MVT RetVT, Register Op0) {
8777 switch (VT.SimpleTy) {
8778 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(RetVT, Op0);
8779 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(RetVT, Op0);
8780 default: return Register();
8781 }
8782}
8783
8784// FastEmit functions for X86ISD::STRICT_VFPROUND.
8785
8786Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
8787 if (RetVT.SimpleTy != MVT::v8f16)
8788 return Register();
8789 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8790 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
8791 }
8792 return Register();
8793}
8794
8795Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(MVT RetVT, Register Op0) {
8796 if (RetVT.SimpleTy != MVT::v8f16)
8797 return Register();
8798 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8799 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
8800 }
8801 return Register();
8802}
8803
8804Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(MVT RetVT, Register Op0) {
8805 if (RetVT.SimpleTy != MVT::v16f16)
8806 return Register();
8807 if ((Subtarget->hasFP16())) {
8808 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
8809 }
8810 return Register();
8811}
8812
8813Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Register Op0) {
8814 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8815 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
8816 }
8817 return Register();
8818}
8819
8820Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Register Op0) {
8821 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8822 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
8823 }
8824 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8825 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8826 }
8827 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8828 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
8829 }
8830 return Register();
8831}
8832
8833Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
8834switch (RetVT.SimpleTy) {
8835 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
8836 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
8837 default: return Register();
8838}
8839}
8840
8841Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Register Op0) {
8842 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8843 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
8844 }
8845 return Register();
8846}
8847
8848Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Register Op0) {
8849 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8850 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
8851 }
8852 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8853 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
8854 }
8855 return Register();
8856}
8857
8858Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(MVT RetVT, Register Op0) {
8859switch (RetVT.SimpleTy) {
8860 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
8861 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
8862 default: return Register();
8863}
8864}
8865
8866Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Register Op0) {
8867 if ((Subtarget->hasFP16())) {
8868 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
8869 }
8870 return Register();
8871}
8872
8873Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Register Op0) {
8874 if ((Subtarget->hasAVX512())) {
8875 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
8876 }
8877 return Register();
8878}
8879
8880Register fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(MVT RetVT, Register Op0) {
8881switch (RetVT.SimpleTy) {
8882 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
8883 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
8884 default: return Register();
8885}
8886}
8887
8888Register fastEmit_X86ISD_STRICT_VFPROUND_r(MVT VT, MVT RetVT, Register Op0) {
8889 switch (VT.SimpleTy) {
8890 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(RetVT, Op0);
8891 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(RetVT, Op0);
8892 case MVT::v16f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(RetVT, Op0);
8893 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(RetVT, Op0);
8894 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(RetVT, Op0);
8895 case MVT::v8f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(RetVT, Op0);
8896 default: return Register();
8897 }
8898}
8899
8900// FastEmit functions for X86ISD::VBROADCAST.
8901
8902Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Register Op0) {
8903 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8904 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ128rr, RC: &X86::VR128XRegClass, Op0);
8905 }
8906 return Register();
8907}
8908
8909Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Register Op0) {
8910 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8911 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZ256rr, RC: &X86::VR256XRegClass, Op0);
8912 }
8913 return Register();
8914}
8915
8916Register fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Register Op0) {
8917 if ((Subtarget->hasAVX512())) {
8918 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrZrr, RC: &X86::VR512RegClass, Op0);
8919 }
8920 return Register();
8921}
8922
8923Register fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, Register Op0) {
8924switch (RetVT.SimpleTy) {
8925 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0);
8926 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0);
8927 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0);
8928 default: return Register();
8929}
8930}
8931
8932Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Register Op0) {
8933 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8934 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ128rr, RC: &X86::VR128XRegClass, Op0);
8935 }
8936 return Register();
8937}
8938
8939Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Register Op0) {
8940 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8941 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZ256rr, RC: &X86::VR256XRegClass, Op0);
8942 }
8943 return Register();
8944}
8945
8946Register fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Register Op0) {
8947 if ((Subtarget->hasAVX512())) {
8948 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrZrr, RC: &X86::VR512RegClass, Op0);
8949 }
8950 return Register();
8951}
8952
8953Register fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, Register Op0) {
8954switch (RetVT.SimpleTy) {
8955 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0);
8956 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0);
8957 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0);
8958 default: return Register();
8959}
8960}
8961
8962Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Register Op0) {
8963 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8964 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ128rr, RC: &X86::VR128XRegClass, Op0);
8965 }
8966 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8967 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBrr, RC: &X86::VR128RegClass, Op0);
8968 }
8969 return Register();
8970}
8971
8972Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Register Op0) {
8973 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8974 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZ256rr, RC: &X86::VR256XRegClass, Op0);
8975 }
8976 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8977 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBYrr, RC: &X86::VR256RegClass, Op0);
8978 }
8979 return Register();
8980}
8981
8982Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Register Op0) {
8983 if ((Subtarget->hasBWI())) {
8984 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTBZrr, RC: &X86::VR512RegClass, Op0);
8985 }
8986 return Register();
8987}
8988
8989Register fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
8990switch (RetVT.SimpleTy) {
8991 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0);
8992 case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0);
8993 case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0);
8994 default: return Register();
8995}
8996}
8997
8998Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Register Op0) {
8999 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9000 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
9001 }
9002 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9003 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
9004 }
9005 return Register();
9006}
9007
9008Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Register Op0) {
9009 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9010 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
9011 }
9012 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9013 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
9014 }
9015 return Register();
9016}
9017
9018Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Register Op0) {
9019 if ((Subtarget->hasBWI())) {
9020 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
9021 }
9022 return Register();
9023}
9024
9025Register fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
9026switch (RetVT.SimpleTy) {
9027 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0);
9028 case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0);
9029 case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0);
9030 default: return Register();
9031}
9032}
9033
9034Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Register Op0) {
9035 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9036 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ128rr, RC: &X86::VR128XRegClass, Op0);
9037 }
9038 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9039 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDrr, RC: &X86::VR128RegClass, Op0);
9040 }
9041 return Register();
9042}
9043
9044Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Register Op0) {
9045 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9046 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZ256rr, RC: &X86::VR256XRegClass, Op0);
9047 }
9048 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9049 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDYrr, RC: &X86::VR256RegClass, Op0);
9050 }
9051 return Register();
9052}
9053
9054Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Register Op0) {
9055 if ((Subtarget->hasAVX512())) {
9056 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTDZrr, RC: &X86::VR512RegClass, Op0);
9057 }
9058 return Register();
9059}
9060
9061Register fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
9062switch (RetVT.SimpleTy) {
9063 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0);
9064 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0);
9065 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0);
9066 default: return Register();
9067}
9068}
9069
9070Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Register Op0) {
9071 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9072 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ128rr, RC: &X86::VR128XRegClass, Op0);
9073 }
9074 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9075 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQrr, RC: &X86::VR128RegClass, Op0);
9076 }
9077 return Register();
9078}
9079
9080Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Register Op0) {
9081 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9082 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZ256rr, RC: &X86::VR256XRegClass, Op0);
9083 }
9084 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9085 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQYrr, RC: &X86::VR256RegClass, Op0);
9086 }
9087 return Register();
9088}
9089
9090Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Register Op0) {
9091 if ((Subtarget->hasAVX512())) {
9092 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTQZrr, RC: &X86::VR512RegClass, Op0);
9093 }
9094 return Register();
9095}
9096
9097Register fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
9098switch (RetVT.SimpleTy) {
9099 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0);
9100 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0);
9101 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0);
9102 default: return Register();
9103}
9104}
9105
9106Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Register Op0) {
9107 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9108 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
9109 }
9110 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9111 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWrr, RC: &X86::VR128RegClass, Op0);
9112 }
9113 return Register();
9114}
9115
9116Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Register Op0) {
9117 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9118 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
9119 }
9120 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9121 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWYrr, RC: &X86::VR256RegClass, Op0);
9122 }
9123 return Register();
9124}
9125
9126Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Register Op0) {
9127 if ((Subtarget->hasBWI())) {
9128 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
9129 }
9130 return Register();
9131}
9132
9133Register fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
9134switch (RetVT.SimpleTy) {
9135 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Op0);
9136 case MVT::v16f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Op0);
9137 case MVT::v32f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Op0);
9138 default: return Register();
9139}
9140}
9141
9142Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Register Op0) {
9143 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9144 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ128rr, RC: &X86::VR128XRegClass, Op0);
9145 }
9146 return Register();
9147}
9148
9149Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Register Op0) {
9150 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9151 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZ256rr, RC: &X86::VR256XRegClass, Op0);
9152 }
9153 return Register();
9154}
9155
9156Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Register Op0) {
9157 if ((Subtarget->hasBF16())) {
9158 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTWZrr, RC: &X86::VR512RegClass, Op0);
9159 }
9160 return Register();
9161}
9162
9163Register fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
9164switch (RetVT.SimpleTy) {
9165 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Op0);
9166 case MVT::v16bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Op0);
9167 case MVT::v32bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Op0);
9168 default: return Register();
9169}
9170}
9171
9172Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Register Op0) {
9173 if ((Subtarget->hasVLX())) {
9174 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ128rr, RC: &X86::VR128XRegClass, Op0);
9175 }
9176 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9177 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSrr, RC: &X86::VR128RegClass, Op0);
9178 }
9179 return Register();
9180}
9181
9182Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Register Op0) {
9183 if ((Subtarget->hasVLX())) {
9184 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZ256rr, RC: &X86::VR256XRegClass, Op0);
9185 }
9186 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9187 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSYrr, RC: &X86::VR256RegClass, Op0);
9188 }
9189 return Register();
9190}
9191
9192Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Register Op0) {
9193 if ((Subtarget->hasAVX512())) {
9194 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSSZrr, RC: &X86::VR512RegClass, Op0);
9195 }
9196 return Register();
9197}
9198
9199Register fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
9200switch (RetVT.SimpleTy) {
9201 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0);
9202 case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0);
9203 case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0);
9204 default: return Register();
9205}
9206}
9207
9208Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Register Op0) {
9209 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9210 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPrr, RC: &X86::VR128RegClass, Op0);
9211 }
9212 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9213 return fastEmitInst_r(MachineInstOpcode: X86::VMOVDDUPZ128rr, RC: &X86::VR128XRegClass, Op0);
9214 }
9215 return Register();
9216}
9217
9218Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Register Op0) {
9219 if ((Subtarget->hasVLX())) {
9220 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZ256rr, RC: &X86::VR256XRegClass, Op0);
9221 }
9222 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9223 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDYrr, RC: &X86::VR256RegClass, Op0);
9224 }
9225 return Register();
9226}
9227
9228Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Register Op0) {
9229 if ((Subtarget->hasAVX512())) {
9230 return fastEmitInst_r(MachineInstOpcode: X86::VBROADCASTSDZrr, RC: &X86::VR512RegClass, Op0);
9231 }
9232 return Register();
9233}
9234
9235Register fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
9236switch (RetVT.SimpleTy) {
9237 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0);
9238 case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0);
9239 case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0);
9240 default: return Register();
9241}
9242}
9243
9244Register fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, Register Op0) {
9245 switch (VT.SimpleTy) {
9246 case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0);
9247 case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0);
9248 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0);
9249 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0);
9250 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0);
9251 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0);
9252 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(RetVT, Op0);
9253 case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(RetVT, Op0);
9254 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0);
9255 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0);
9256 default: return Register();
9257 }
9258}
9259
9260// FastEmit functions for X86ISD::VBROADCASTM.
9261
9262Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Register Op0) {
9263 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9264 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ128rr, RC: &X86::VR128XRegClass, Op0);
9265 }
9266 return Register();
9267}
9268
9269Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Register Op0) {
9270 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9271 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZ256rr, RC: &X86::VR256XRegClass, Op0);
9272 }
9273 return Register();
9274}
9275
9276Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Register Op0) {
9277 if ((Subtarget->hasCDI())) {
9278 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMB2QZrr, RC: &X86::VR512RegClass, Op0);
9279 }
9280 return Register();
9281}
9282
9283Register fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, Register Op0) {
9284switch (RetVT.SimpleTy) {
9285 case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0);
9286 case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0);
9287 case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0);
9288 default: return Register();
9289}
9290}
9291
9292Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Register Op0) {
9293 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9294 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ128rr, RC: &X86::VR128XRegClass, Op0);
9295 }
9296 return Register();
9297}
9298
9299Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Register Op0) {
9300 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
9301 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZ256rr, RC: &X86::VR256XRegClass, Op0);
9302 }
9303 return Register();
9304}
9305
9306Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Register Op0) {
9307 if ((Subtarget->hasCDI())) {
9308 return fastEmitInst_r(MachineInstOpcode: X86::VPBROADCASTMW2DZrr, RC: &X86::VR512RegClass, Op0);
9309 }
9310 return Register();
9311}
9312
9313Register fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, Register Op0) {
9314switch (RetVT.SimpleTy) {
9315 case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0);
9316 case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0);
9317 case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0);
9318 default: return Register();
9319}
9320}
9321
9322Register fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, Register Op0) {
9323 switch (VT.SimpleTy) {
9324 case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0);
9325 case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0);
9326 default: return Register();
9327 }
9328}
9329
9330// FastEmit functions for X86ISD::VCVTHF82PH.
9331
9332Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v8f16_r(Register Op0) {
9333 if ((Subtarget->hasAVX10_2())) {
9334 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZ128rr, RC: &X86::VR128XRegClass, Op0);
9335 }
9336 return Register();
9337}
9338
9339Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v16f16_r(Register Op0) {
9340 if ((Subtarget->hasAVX10_2())) {
9341 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZ256rr, RC: &X86::VR256XRegClass, Op0);
9342 }
9343 return Register();
9344}
9345
9346Register fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_r(MVT RetVT, Register Op0) {
9347switch (RetVT.SimpleTy) {
9348 case MVT::v8f16: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v8f16_r(Op0);
9349 case MVT::v16f16: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_MVT_v16f16_r(Op0);
9350 default: return Register();
9351}
9352}
9353
9354Register fastEmit_X86ISD_VCVTHF82PH_MVT_v32i8_r(MVT RetVT, Register Op0) {
9355 if (RetVT.SimpleTy != MVT::v32f16)
9356 return Register();
9357 if ((Subtarget->hasAVX10_2())) {
9358 return fastEmitInst_r(MachineInstOpcode: X86::VCVTHF82PHZrr, RC: &X86::VR512RegClass, Op0);
9359 }
9360 return Register();
9361}
9362
9363Register fastEmit_X86ISD_VCVTHF82PH_r(MVT VT, MVT RetVT, Register Op0) {
9364 switch (VT.SimpleTy) {
9365 case MVT::v16i8: return fastEmit_X86ISD_VCVTHF82PH_MVT_v16i8_r(RetVT, Op0);
9366 case MVT::v32i8: return fastEmit_X86ISD_VCVTHF82PH_MVT_v32i8_r(RetVT, Op0);
9367 default: return Register();
9368 }
9369}
9370
9371// FastEmit functions for X86ISD::VCVTPH2BF8.
9372
9373Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v8f16_r(MVT RetVT, Register Op0) {
9374 if (RetVT.SimpleTy != MVT::v16i8)
9375 return Register();
9376 if ((Subtarget->hasAVX10_2())) {
9377 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9378 }
9379 return Register();
9380}
9381
9382Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v16f16_r(MVT RetVT, Register Op0) {
9383 if (RetVT.SimpleTy != MVT::v16i8)
9384 return Register();
9385 if ((Subtarget->hasAVX10_2())) {
9386 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9387 }
9388 return Register();
9389}
9390
9391Register fastEmit_X86ISD_VCVTPH2BF8_MVT_v32f16_r(MVT RetVT, Register Op0) {
9392 if (RetVT.SimpleTy != MVT::v32i8)
9393 return Register();
9394 if ((Subtarget->hasAVX10_2())) {
9395 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8Zrr, RC: &X86::VR256XRegClass, Op0);
9396 }
9397 return Register();
9398}
9399
9400Register fastEmit_X86ISD_VCVTPH2BF8_r(MVT VT, MVT RetVT, Register Op0) {
9401 switch (VT.SimpleTy) {
9402 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v8f16_r(RetVT, Op0);
9403 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v16f16_r(RetVT, Op0);
9404 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2BF8_MVT_v32f16_r(RetVT, Op0);
9405 default: return Register();
9406 }
9407}
9408
9409// FastEmit functions for X86ISD::VCVTPH2BF8S.
9410
9411Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v8f16_r(MVT RetVT, Register Op0) {
9412 if (RetVT.SimpleTy != MVT::v16i8)
9413 return Register();
9414 if ((Subtarget->hasAVX10_2())) {
9415 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9416 }
9417 return Register();
9418}
9419
9420Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v16f16_r(MVT RetVT, Register Op0) {
9421 if (RetVT.SimpleTy != MVT::v16i8)
9422 return Register();
9423 if ((Subtarget->hasAVX10_2())) {
9424 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9425 }
9426 return Register();
9427}
9428
9429Register fastEmit_X86ISD_VCVTPH2BF8S_MVT_v32f16_r(MVT RetVT, Register Op0) {
9430 if (RetVT.SimpleTy != MVT::v32i8)
9431 return Register();
9432 if ((Subtarget->hasAVX10_2())) {
9433 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2BF8SZrr, RC: &X86::VR256XRegClass, Op0);
9434 }
9435 return Register();
9436}
9437
9438Register fastEmit_X86ISD_VCVTPH2BF8S_r(MVT VT, MVT RetVT, Register Op0) {
9439 switch (VT.SimpleTy) {
9440 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v8f16_r(RetVT, Op0);
9441 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v16f16_r(RetVT, Op0);
9442 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2BF8S_MVT_v32f16_r(RetVT, Op0);
9443 default: return Register();
9444 }
9445}
9446
9447// FastEmit functions for X86ISD::VCVTPH2HF8.
9448
9449Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v8f16_r(MVT RetVT, Register Op0) {
9450 if (RetVT.SimpleTy != MVT::v16i8)
9451 return Register();
9452 if ((Subtarget->hasAVX10_2())) {
9453 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Z128rr, RC: &X86::VR128XRegClass, Op0);
9454 }
9455 return Register();
9456}
9457
9458Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v16f16_r(MVT RetVT, Register Op0) {
9459 if (RetVT.SimpleTy != MVT::v16i8)
9460 return Register();
9461 if ((Subtarget->hasAVX10_2())) {
9462 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Z256rr, RC: &X86::VR128XRegClass, Op0);
9463 }
9464 return Register();
9465}
9466
9467Register fastEmit_X86ISD_VCVTPH2HF8_MVT_v32f16_r(MVT RetVT, Register Op0) {
9468 if (RetVT.SimpleTy != MVT::v32i8)
9469 return Register();
9470 if ((Subtarget->hasAVX10_2())) {
9471 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8Zrr, RC: &X86::VR256XRegClass, Op0);
9472 }
9473 return Register();
9474}
9475
9476Register fastEmit_X86ISD_VCVTPH2HF8_r(MVT VT, MVT RetVT, Register Op0) {
9477 switch (VT.SimpleTy) {
9478 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v8f16_r(RetVT, Op0);
9479 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v16f16_r(RetVT, Op0);
9480 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2HF8_MVT_v32f16_r(RetVT, Op0);
9481 default: return Register();
9482 }
9483}
9484
9485// FastEmit functions for X86ISD::VCVTPH2HF8S.
9486
9487Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v8f16_r(MVT RetVT, Register Op0) {
9488 if (RetVT.SimpleTy != MVT::v16i8)
9489 return Register();
9490 if ((Subtarget->hasAVX10_2())) {
9491 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0);
9492 }
9493 return Register();
9494}
9495
9496Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v16f16_r(MVT RetVT, Register Op0) {
9497 if (RetVT.SimpleTy != MVT::v16i8)
9498 return Register();
9499 if ((Subtarget->hasAVX10_2())) {
9500 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZ256rr, RC: &X86::VR128XRegClass, Op0);
9501 }
9502 return Register();
9503}
9504
9505Register fastEmit_X86ISD_VCVTPH2HF8S_MVT_v32f16_r(MVT RetVT, Register Op0) {
9506 if (RetVT.SimpleTy != MVT::v32i8)
9507 return Register();
9508 if ((Subtarget->hasAVX10_2())) {
9509 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2HF8SZrr, RC: &X86::VR256XRegClass, Op0);
9510 }
9511 return Register();
9512}
9513
9514Register fastEmit_X86ISD_VCVTPH2HF8S_r(MVT VT, MVT RetVT, Register Op0) {
9515 switch (VT.SimpleTy) {
9516 case MVT::v8f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v8f16_r(RetVT, Op0);
9517 case MVT::v16f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v16f16_r(RetVT, Op0);
9518 case MVT::v32f16: return fastEmit_X86ISD_VCVTPH2HF8S_MVT_v32f16_r(RetVT, Op0);
9519 default: return Register();
9520 }
9521}
9522
9523// FastEmit functions for X86ISD::VFPEXT.
9524
9525Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Register Op0) {
9526 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9527 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZ128rr, RC: &X86::VR128XRegClass, Op0);
9528 }
9529 return Register();
9530}
9531
9532Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Register Op0) {
9533 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9534 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
9535 }
9536 return Register();
9537}
9538
9539Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Register Op0) {
9540 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9541 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZ256rr, RC: &X86::VR256XRegClass, Op0);
9542 }
9543 return Register();
9544}
9545
9546Register fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(MVT RetVT, Register Op0) {
9547switch (RetVT.SimpleTy) {
9548 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
9549 case MVT::v2f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
9550 case MVT::v4f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
9551 default: return Register();
9552}
9553}
9554
9555Register fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, Register Op0) {
9556 if (RetVT.SimpleTy != MVT::v2f64)
9557 return Register();
9558 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9559 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZ128rr, RC: &X86::VR128XRegClass, Op0);
9560 }
9561 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9562 return fastEmitInst_r(MachineInstOpcode: X86::CVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
9563 }
9564 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9565 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDrr, RC: &X86::VR128RegClass, Op0);
9566 }
9567 return Register();
9568}
9569
9570Register fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, Register Op0) {
9571 switch (VT.SimpleTy) {
9572 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(RetVT, Op0);
9573 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0);
9574 default: return Register();
9575 }
9576}
9577
9578// FastEmit functions for X86ISD::VFPEXT_SAE.
9579
9580Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(MVT RetVT, Register Op0) {
9581 if (RetVT.SimpleTy != MVT::v8f64)
9582 return Register();
9583 if ((Subtarget->hasFP16())) {
9584 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PDZrrb, RC: &X86::VR512RegClass, Op0);
9585 }
9586 return Register();
9587}
9588
9589Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(MVT RetVT, Register Op0) {
9590 if (RetVT.SimpleTy != MVT::v16f32)
9591 return Register();
9592 if ((Subtarget->hasFP16())) {
9593 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPH2PSXZrrb, RC: &X86::VR512RegClass, Op0);
9594 }
9595 return Register();
9596}
9597
9598Register fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(MVT RetVT, Register Op0) {
9599 if (RetVT.SimpleTy != MVT::v8f64)
9600 return Register();
9601 if ((Subtarget->hasAVX512())) {
9602 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PDZrrb, RC: &X86::VR512RegClass, Op0);
9603 }
9604 return Register();
9605}
9606
9607Register fastEmit_X86ISD_VFPEXT_SAE_r(MVT VT, MVT RetVT, Register Op0) {
9608 switch (VT.SimpleTy) {
9609 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(RetVT, Op0);
9610 case MVT::v16f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(RetVT, Op0);
9611 case MVT::v8f32: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(RetVT, Op0);
9612 default: return Register();
9613 }
9614}
9615
9616// FastEmit functions for X86ISD::VFPROUND.
9617
9618Register fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
9619 if (RetVT.SimpleTy != MVT::v8f16)
9620 return Register();
9621 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9622 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0);
9623 }
9624 return Register();
9625}
9626
9627Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Register Op0) {
9628 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9629 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZ256rr, RC: &X86::VR128XRegClass, Op0);
9630 }
9631 return Register();
9632}
9633
9634Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Register Op0) {
9635 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
9636 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Z256rr, RC: &X86::VR128XRegClass, Op0);
9637 }
9638 if ((Subtarget->hasAVXNECONVERT())) {
9639 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Yrr, RC: &X86::VR128RegClass, Op0);
9640 }
9641 return Register();
9642}
9643
9644Register fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(MVT RetVT, Register Op0) {
9645switch (RetVT.SimpleTy) {
9646 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Op0);
9647 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Op0);
9648 default: return Register();
9649}
9650}
9651
9652Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Register Op0) {
9653 if ((Subtarget->hasFP16())) {
9654 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPS2PHXZrr, RC: &X86::VR256XRegClass, Op0);
9655 }
9656 return Register();
9657}
9658
9659Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Register Op0) {
9660 if ((Subtarget->hasBF16())) {
9661 return fastEmitInst_r(MachineInstOpcode: X86::VCVTNEPS2BF16Zrr, RC: &X86::VR256XRegClass, Op0);
9662 }
9663 return Register();
9664}
9665
9666Register fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(MVT RetVT, Register Op0) {
9667switch (RetVT.SimpleTy) {
9668 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Op0);
9669 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Op0);
9670 default: return Register();
9671}
9672}
9673
9674Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Register Op0) {
9675 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9676 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ128rr, RC: &X86::VR128XRegClass, Op0);
9677 }
9678 return Register();
9679}
9680
9681Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Register Op0) {
9682 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9683 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ128rr, RC: &X86::VR128XRegClass, Op0);
9684 }
9685 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9686 return fastEmitInst_r(MachineInstOpcode: X86::CVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
9687 }
9688 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9689 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSrr, RC: &X86::VR128RegClass, Op0);
9690 }
9691 return Register();
9692}
9693
9694Register fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
9695switch (RetVT.SimpleTy) {
9696 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
9697 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
9698 default: return Register();
9699}
9700}
9701
9702Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Register Op0) {
9703 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9704 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZ256rr, RC: &X86::VR128XRegClass, Op0);
9705 }
9706 return Register();
9707}
9708
9709Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Register Op0) {
9710 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9711 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZ256rr, RC: &X86::VR128XRegClass, Op0);
9712 }
9713 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9714 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSYrr, RC: &X86::VR128RegClass, Op0);
9715 }
9716 return Register();
9717}
9718
9719Register fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(MVT RetVT, Register Op0) {
9720switch (RetVT.SimpleTy) {
9721 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
9722 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
9723 default: return Register();
9724}
9725}
9726
9727Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Register Op0) {
9728 if ((Subtarget->hasFP16())) {
9729 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PHZrr, RC: &X86::VR128XRegClass, Op0);
9730 }
9731 return Register();
9732}
9733
9734Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Register Op0) {
9735 if ((Subtarget->hasAVX512())) {
9736 return fastEmitInst_r(MachineInstOpcode: X86::VCVTPD2PSZrr, RC: &X86::VR256XRegClass, Op0);
9737 }
9738 return Register();
9739}
9740
9741Register fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(MVT RetVT, Register Op0) {
9742switch (RetVT.SimpleTy) {
9743 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
9744 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
9745 default: return Register();
9746}
9747}
9748
9749Register fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, Register Op0) {
9750 switch (VT.SimpleTy) {
9751 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(RetVT, Op0);
9752 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(RetVT, Op0);
9753 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(RetVT, Op0);
9754 case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0);
9755 case MVT::v4f64: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(RetVT, Op0);
9756 case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(RetVT, Op0);
9757 default: return Register();
9758 }
9759}
9760
9761// FastEmit functions for X86ISD::VTRUNC.
9762
9763Register fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, Register Op0) {
9764 if (RetVT.SimpleTy != MVT::v16i8)
9765 return Register();
9766 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9767 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVWBZ128rr, RC: &X86::VR128XRegClass, Op0);
9768 }
9769 return Register();
9770}
9771
9772Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Register Op0) {
9773 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9774 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ128rr, RC: &X86::VR128XRegClass, Op0);
9775 }
9776 return Register();
9777}
9778
9779Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Register Op0) {
9780 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9781 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDWZ128rr, RC: &X86::VR128XRegClass, Op0);
9782 }
9783 return Register();
9784}
9785
9786Register fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, Register Op0) {
9787switch (RetVT.SimpleTy) {
9788 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0);
9789 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0);
9790 default: return Register();
9791}
9792}
9793
9794Register fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, Register Op0) {
9795 if (RetVT.SimpleTy != MVT::v16i8)
9796 return Register();
9797 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9798 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVDBZ256rr, RC: &X86::VR128XRegClass, Op0);
9799 }
9800 return Register();
9801}
9802
9803Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Register Op0) {
9804 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9805 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ128rr, RC: &X86::VR128XRegClass, Op0);
9806 }
9807 return Register();
9808}
9809
9810Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Register Op0) {
9811 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9812 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ128rr, RC: &X86::VR128XRegClass, Op0);
9813 }
9814 return Register();
9815}
9816
9817Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Register Op0) {
9818 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9819 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQDZ128rr, RC: &X86::VR128XRegClass, Op0);
9820 }
9821 return Register();
9822}
9823
9824Register fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, Register Op0) {
9825switch (RetVT.SimpleTy) {
9826 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0);
9827 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0);
9828 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0);
9829 default: return Register();
9830}
9831}
9832
9833Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Register Op0) {
9834 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9835 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZ256rr, RC: &X86::VR128XRegClass, Op0);
9836 }
9837 return Register();
9838}
9839
9840Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Register Op0) {
9841 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9842 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQWZ256rr, RC: &X86::VR128XRegClass, Op0);
9843 }
9844 return Register();
9845}
9846
9847Register fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, Register Op0) {
9848switch (RetVT.SimpleTy) {
9849 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0);
9850 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0);
9851 default: return Register();
9852}
9853}
9854
9855Register fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, Register Op0) {
9856 if (RetVT.SimpleTy != MVT::v16i8)
9857 return Register();
9858 if ((Subtarget->hasAVX512())) {
9859 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVQBZrr, RC: &X86::VR128XRegClass, Op0);
9860 }
9861 return Register();
9862}
9863
9864Register fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
9865 switch (VT.SimpleTy) {
9866 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0);
9867 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0);
9868 case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0);
9869 case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0);
9870 case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0);
9871 case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0);
9872 default: return Register();
9873 }
9874}
9875
9876// FastEmit functions for X86ISD::VTRUNCS.
9877
9878Register fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, Register Op0) {
9879 if (RetVT.SimpleTy != MVT::v16i8)
9880 return Register();
9881 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9882 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
9883 }
9884 return Register();
9885}
9886
9887Register fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, Register Op0) {
9888 if (RetVT.SimpleTy != MVT::v16i8)
9889 return Register();
9890 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9891 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
9892 }
9893 return Register();
9894}
9895
9896Register fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, Register Op0) {
9897 if (RetVT.SimpleTy != MVT::v32i8)
9898 return Register();
9899 if ((Subtarget->hasBWI())) {
9900 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSWBZrr, RC: &X86::VR256XRegClass, Op0);
9901 }
9902 return Register();
9903}
9904
9905Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Register Op0) {
9906 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9907 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
9908 }
9909 return Register();
9910}
9911
9912Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Register Op0) {
9913 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9914 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
9915 }
9916 return Register();
9917}
9918
9919Register fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, Register Op0) {
9920switch (RetVT.SimpleTy) {
9921 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0);
9922 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0);
9923 default: return Register();
9924}
9925}
9926
9927Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Register Op0) {
9928 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9929 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
9930 }
9931 return Register();
9932}
9933
9934Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Register Op0) {
9935 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9936 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
9937 }
9938 return Register();
9939}
9940
9941Register fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, Register Op0) {
9942switch (RetVT.SimpleTy) {
9943 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0);
9944 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0);
9945 default: return Register();
9946}
9947}
9948
9949Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Register Op0) {
9950 if ((Subtarget->hasAVX512())) {
9951 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDBZrr, RC: &X86::VR128XRegClass, Op0);
9952 }
9953 return Register();
9954}
9955
9956Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Register Op0) {
9957 if ((Subtarget->hasAVX512())) {
9958 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSDWZrr, RC: &X86::VR256XRegClass, Op0);
9959 }
9960 return Register();
9961}
9962
9963Register fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, Register Op0) {
9964switch (RetVT.SimpleTy) {
9965 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0);
9966 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0);
9967 default: return Register();
9968}
9969}
9970
9971Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Register Op0) {
9972 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9973 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
9974 }
9975 return Register();
9976}
9977
9978Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Register Op0) {
9979 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9980 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
9981 }
9982 return Register();
9983}
9984
9985Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Register Op0) {
9986 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9987 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
9988 }
9989 return Register();
9990}
9991
9992Register fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, Register Op0) {
9993switch (RetVT.SimpleTy) {
9994 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0);
9995 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0);
9996 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0);
9997 default: return Register();
9998}
9999}
10000
10001Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Register Op0) {
10002 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10003 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
10004 }
10005 return Register();
10006}
10007
10008Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Register Op0) {
10009 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10010 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
10011 }
10012 return Register();
10013}
10014
10015Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Register Op0) {
10016 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10017 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
10018 }
10019 return Register();
10020}
10021
10022Register fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, Register Op0) {
10023switch (RetVT.SimpleTy) {
10024 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0);
10025 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0);
10026 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0);
10027 default: return Register();
10028}
10029}
10030
10031Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Register Op0) {
10032 if ((Subtarget->hasAVX512())) {
10033 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQBZrr, RC: &X86::VR128XRegClass, Op0);
10034 }
10035 return Register();
10036}
10037
10038Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Register Op0) {
10039 if ((Subtarget->hasAVX512())) {
10040 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQWZrr, RC: &X86::VR128XRegClass, Op0);
10041 }
10042 return Register();
10043}
10044
10045Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Register Op0) {
10046 if ((Subtarget->hasAVX512())) {
10047 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVSQDZrr, RC: &X86::VR256XRegClass, Op0);
10048 }
10049 return Register();
10050}
10051
10052Register fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, Register Op0) {
10053switch (RetVT.SimpleTy) {
10054 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0);
10055 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0);
10056 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0);
10057 default: return Register();
10058}
10059}
10060
10061Register fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, Register Op0) {
10062 switch (VT.SimpleTy) {
10063 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0);
10064 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0);
10065 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0);
10066 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0);
10067 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0);
10068 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0);
10069 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0);
10070 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0);
10071 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0);
10072 default: return Register();
10073 }
10074}
10075
10076// FastEmit functions for X86ISD::VTRUNCUS.
10077
10078Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, Register Op0) {
10079 if (RetVT.SimpleTy != MVT::v16i8)
10080 return Register();
10081 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10082 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ128rr, RC: &X86::VR128XRegClass, Op0);
10083 }
10084 return Register();
10085}
10086
10087Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, Register Op0) {
10088 if (RetVT.SimpleTy != MVT::v16i8)
10089 return Register();
10090 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10091 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZ256rr, RC: &X86::VR128XRegClass, Op0);
10092 }
10093 return Register();
10094}
10095
10096Register fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, Register Op0) {
10097 if (RetVT.SimpleTy != MVT::v32i8)
10098 return Register();
10099 if ((Subtarget->hasBWI())) {
10100 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSWBZrr, RC: &X86::VR256XRegClass, Op0);
10101 }
10102 return Register();
10103}
10104
10105Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Register Op0) {
10106 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10107 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ128rr, RC: &X86::VR128XRegClass, Op0);
10108 }
10109 return Register();
10110}
10111
10112Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Register Op0) {
10113 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10114 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ128rr, RC: &X86::VR128XRegClass, Op0);
10115 }
10116 return Register();
10117}
10118
10119Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, Register Op0) {
10120switch (RetVT.SimpleTy) {
10121 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0);
10122 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0);
10123 default: return Register();
10124}
10125}
10126
10127Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Register Op0) {
10128 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10129 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZ256rr, RC: &X86::VR128XRegClass, Op0);
10130 }
10131 return Register();
10132}
10133
10134Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Register Op0) {
10135 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10136 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZ256rr, RC: &X86::VR128XRegClass, Op0);
10137 }
10138 return Register();
10139}
10140
10141Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, Register Op0) {
10142switch (RetVT.SimpleTy) {
10143 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0);
10144 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0);
10145 default: return Register();
10146}
10147}
10148
10149Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Register Op0) {
10150 if ((Subtarget->hasAVX512())) {
10151 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDBZrr, RC: &X86::VR128XRegClass, Op0);
10152 }
10153 return Register();
10154}
10155
10156Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Register Op0) {
10157 if ((Subtarget->hasAVX512())) {
10158 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSDWZrr, RC: &X86::VR256XRegClass, Op0);
10159 }
10160 return Register();
10161}
10162
10163Register fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, Register Op0) {
10164switch (RetVT.SimpleTy) {
10165 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0);
10166 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0);
10167 default: return Register();
10168}
10169}
10170
10171Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Register Op0) {
10172 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10173 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ128rr, RC: &X86::VR128XRegClass, Op0);
10174 }
10175 return Register();
10176}
10177
10178Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Register Op0) {
10179 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10180 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ128rr, RC: &X86::VR128XRegClass, Op0);
10181 }
10182 return Register();
10183}
10184
10185Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Register Op0) {
10186 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10187 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ128rr, RC: &X86::VR128XRegClass, Op0);
10188 }
10189 return Register();
10190}
10191
10192Register fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, Register Op0) {
10193switch (RetVT.SimpleTy) {
10194 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0);
10195 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0);
10196 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0);
10197 default: return Register();
10198}
10199}
10200
10201Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Register Op0) {
10202 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10203 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZ256rr, RC: &X86::VR128XRegClass, Op0);
10204 }
10205 return Register();
10206}
10207
10208Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Register Op0) {
10209 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10210 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZ256rr, RC: &X86::VR128XRegClass, Op0);
10211 }
10212 return Register();
10213}
10214
10215Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Register Op0) {
10216 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10217 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZ256rr, RC: &X86::VR128XRegClass, Op0);
10218 }
10219 return Register();
10220}
10221
10222Register fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, Register Op0) {
10223switch (RetVT.SimpleTy) {
10224 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0);
10225 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0);
10226 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0);
10227 default: return Register();
10228}
10229}
10230
10231Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Register Op0) {
10232 if ((Subtarget->hasAVX512())) {
10233 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQBZrr, RC: &X86::VR128XRegClass, Op0);
10234 }
10235 return Register();
10236}
10237
10238Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Register Op0) {
10239 if ((Subtarget->hasAVX512())) {
10240 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQWZrr, RC: &X86::VR128XRegClass, Op0);
10241 }
10242 return Register();
10243}
10244
10245Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Register Op0) {
10246 if ((Subtarget->hasAVX512())) {
10247 return fastEmitInst_r(MachineInstOpcode: X86::VPMOVUSQDZrr, RC: &X86::VR256XRegClass, Op0);
10248 }
10249 return Register();
10250}
10251
10252Register fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, Register Op0) {
10253switch (RetVT.SimpleTy) {
10254 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0);
10255 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0);
10256 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0);
10257 default: return Register();
10258}
10259}
10260
10261Register fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, Register Op0) {
10262 switch (VT.SimpleTy) {
10263 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0);
10264 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0);
10265 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0);
10266 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0);
10267 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0);
10268 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0);
10269 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0);
10270 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0);
10271 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0);
10272 default: return Register();
10273 }
10274}
10275
10276// FastEmit functions for X86ISD::VZEXT_MOVL.
10277
10278Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v8i16_r(MVT RetVT, Register Op0) {
10279 if (RetVT.SimpleTy != MVT::v8i16)
10280 return Register();
10281 if ((Subtarget->hasAVX10_2())) {
10282 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPWILo2PWIZrr, RC: &X86::VR128XRegClass, Op0);
10283 }
10284 return Register();
10285}
10286
10287Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v4i32_r(MVT RetVT, Register Op0) {
10288 if (RetVT.SimpleTy != MVT::v4i32)
10289 return Register();
10290 if ((Subtarget->hasAVX10_2())) {
10291 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPDILo2PDIZrr, RC: &X86::VR128XRegClass, Op0);
10292 }
10293 return Register();
10294}
10295
10296Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, Register Op0) {
10297 if (RetVT.SimpleTy != MVT::v2i64)
10298 return Register();
10299 if ((Subtarget->hasAVX512())) {
10300 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
10301 }
10302 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10303 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10304 }
10305 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10306 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10307 }
10308 return Register();
10309}
10310
10311Register fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, Register Op0) {
10312 if (RetVT.SimpleTy != MVT::v2f64)
10313 return Register();
10314 if ((Subtarget->hasAVX512())) {
10315 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIZrr, RC: &X86::VR128XRegClass, Op0);
10316 }
10317 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10318 return fastEmitInst_r(MachineInstOpcode: X86::MOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10319 }
10320 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10321 return fastEmitInst_r(MachineInstOpcode: X86::VMOVZPQILo2PQIrr, RC: &X86::VR128RegClass, Op0);
10322 }
10323 return Register();
10324}
10325
10326Register fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, Register Op0) {
10327 switch (VT.SimpleTy) {
10328 case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v8i16_r(RetVT, Op0);
10329 case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v4i32_r(RetVT, Op0);
10330 case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0);
10331 case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0);
10332 default: return Register();
10333 }
10334}
10335
10336// Top-level FastEmit function.
10337
10338Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
10339 switch (Opcode) {
10340 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
10341 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
10342 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
10343 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
10344 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
10345 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
10346 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
10347 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
10348 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
10349 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
10350 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
10351 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
10352 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
10353 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
10354 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
10355 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
10356 case ISD::LLRINT: return fastEmit_ISD_LLRINT_r(VT, RetVT, Op0);
10357 case ISD::LRINT: return fastEmit_ISD_LRINT_r(VT, RetVT, Op0);
10358 case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0);
10359 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
10360 case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
10361 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
10362 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
10363 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
10364 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
10365 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
10366 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
10367 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
10368 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
10369 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
10370 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
10371 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
10372 case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
10373 case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0);
10374 case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0);
10375 case X86ISD::CVTNEPS2BF16: return fastEmit_X86ISD_CVTNEPS2BF16_r(VT, RetVT, Op0);
10376 case X86ISD::CVTP2IBS: return fastEmit_X86ISD_CVTP2IBS_r(VT, RetVT, Op0);
10377 case X86ISD::CVTP2IUBS: return fastEmit_X86ISD_CVTP2IUBS_r(VT, RetVT, Op0);
10378 case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0);
10379 case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0);
10380 case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0);
10381 case X86ISD::CVTPH2PS_SAE: return fastEmit_X86ISD_CVTPH2PS_SAE_r(VT, RetVT, Op0);
10382 case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0);
10383 case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0);
10384 case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0);
10385 case X86ISD::CVTTP2IBS: return fastEmit_X86ISD_CVTTP2IBS_r(VT, RetVT, Op0);
10386 case X86ISD::CVTTP2IBS_SAE: return fastEmit_X86ISD_CVTTP2IBS_SAE_r(VT, RetVT, Op0);
10387 case X86ISD::CVTTP2IUBS: return fastEmit_X86ISD_CVTTP2IUBS_r(VT, RetVT, Op0);
10388 case X86ISD::CVTTP2IUBS_SAE: return fastEmit_X86ISD_CVTTP2IUBS_SAE_r(VT, RetVT, Op0);
10389 case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0);
10390 case X86ISD::CVTTP2SIS: return fastEmit_X86ISD_CVTTP2SIS_r(VT, RetVT, Op0);
10391 case X86ISD::CVTTP2SIS_SAE: return fastEmit_X86ISD_CVTTP2SIS_SAE_r(VT, RetVT, Op0);
10392 case X86ISD::CVTTP2SI_SAE: return fastEmit_X86ISD_CVTTP2SI_SAE_r(VT, RetVT, Op0);
10393 case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0);
10394 case X86ISD::CVTTP2UIS: return fastEmit_X86ISD_CVTTP2UIS_r(VT, RetVT, Op0);
10395 case X86ISD::CVTTP2UIS_SAE: return fastEmit_X86ISD_CVTTP2UIS_SAE_r(VT, RetVT, Op0);
10396 case X86ISD::CVTTP2UI_SAE: return fastEmit_X86ISD_CVTTP2UI_SAE_r(VT, RetVT, Op0);
10397 case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0);
10398 case X86ISD::CVTTS2SIS: return fastEmit_X86ISD_CVTTS2SIS_r(VT, RetVT, Op0);
10399 case X86ISD::CVTTS2SIS_SAE: return fastEmit_X86ISD_CVTTS2SIS_SAE_r(VT, RetVT, Op0);
10400 case X86ISD::CVTTS2SI_SAE: return fastEmit_X86ISD_CVTTS2SI_SAE_r(VT, RetVT, Op0);
10401 case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0);
10402 case X86ISD::CVTTS2UIS: return fastEmit_X86ISD_CVTTS2UIS_r(VT, RetVT, Op0);
10403 case X86ISD::CVTTS2UIS_SAE: return fastEmit_X86ISD_CVTTS2UIS_SAE_r(VT, RetVT, Op0);
10404 case X86ISD::CVTTS2UI_SAE: return fastEmit_X86ISD_CVTTS2UI_SAE_r(VT, RetVT, Op0);
10405 case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0);
10406 case X86ISD::DYN_ALLOCA: return fastEmit_X86ISD_DYN_ALLOCA_r(VT, RetVT, Op0);
10407 case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0);
10408 case X86ISD::FGETEXP: return fastEmit_X86ISD_FGETEXP_r(VT, RetVT, Op0);
10409 case X86ISD::FGETEXP_SAE: return fastEmit_X86ISD_FGETEXP_SAE_r(VT, RetVT, Op0);
10410 case X86ISD::FP_TO_SINT_SAT: return fastEmit_X86ISD_FP_TO_SINT_SAT_r(VT, RetVT, Op0);
10411 case X86ISD::FP_TO_UINT_SAT: return fastEmit_X86ISD_FP_TO_UINT_SAT_r(VT, RetVT, Op0);
10412 case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0);
10413 case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0);
10414 case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0);
10415 case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0);
10416 case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0);
10417 case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0);
10418 case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0);
10419 case X86ISD::MOVQ2DQ: return fastEmit_X86ISD_MOVQ2DQ_r(VT, RetVT, Op0);
10420 case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0);
10421 case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0);
10422 case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0);
10423 case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0);
10424 case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0);
10425 case X86ISD::PROBED_ALLOCA: return fastEmit_X86ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
10426 case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0);
10427 case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0);
10428 case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0);
10429 case X86ISD::STRICT_CVTPH2PS: return fastEmit_X86ISD_STRICT_CVTPH2PS_r(VT, RetVT, Op0);
10430 case X86ISD::STRICT_CVTSI2P: return fastEmit_X86ISD_STRICT_CVTSI2P_r(VT, RetVT, Op0);
10431 case X86ISD::STRICT_CVTTP2SI: return fastEmit_X86ISD_STRICT_CVTTP2SI_r(VT, RetVT, Op0);
10432 case X86ISD::STRICT_CVTTP2UI: return fastEmit_X86ISD_STRICT_CVTTP2UI_r(VT, RetVT, Op0);
10433 case X86ISD::STRICT_CVTUI2P: return fastEmit_X86ISD_STRICT_CVTUI2P_r(VT, RetVT, Op0);
10434 case X86ISD::STRICT_VFPEXT: return fastEmit_X86ISD_STRICT_VFPEXT_r(VT, RetVT, Op0);
10435 case X86ISD::STRICT_VFPROUND: return fastEmit_X86ISD_STRICT_VFPROUND_r(VT, RetVT, Op0);
10436 case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0);
10437 case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0);
10438 case X86ISD::VCVTHF82PH: return fastEmit_X86ISD_VCVTHF82PH_r(VT, RetVT, Op0);
10439 case X86ISD::VCVTPH2BF8: return fastEmit_X86ISD_VCVTPH2BF8_r(VT, RetVT, Op0);
10440 case X86ISD::VCVTPH2BF8S: return fastEmit_X86ISD_VCVTPH2BF8S_r(VT, RetVT, Op0);
10441 case X86ISD::VCVTPH2HF8: return fastEmit_X86ISD_VCVTPH2HF8_r(VT, RetVT, Op0);
10442 case X86ISD::VCVTPH2HF8S: return fastEmit_X86ISD_VCVTPH2HF8S_r(VT, RetVT, Op0);
10443 case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0);
10444 case X86ISD::VFPEXT_SAE: return fastEmit_X86ISD_VFPEXT_SAE_r(VT, RetVT, Op0);
10445 case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0);
10446 case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0);
10447 case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0);
10448 case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0);
10449 case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0);
10450 default: return Register();
10451 }
10452}
10453
10454// FastEmit functions for ISD::ADD.
10455
10456Register fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
10457 if (RetVT.SimpleTy != MVT::i8)
10458 return Register();
10459 if ((Subtarget->hasNDD())) {
10460 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10461 }
10462 if ((!Subtarget->hasNDD())) {
10463 return fastEmitInst_rr(MachineInstOpcode: X86::ADD8rr, RC: &X86::GR8RegClass, Op0, Op1);
10464 }
10465 return Register();
10466}
10467
10468Register fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
10469 if (RetVT.SimpleTy != MVT::i16)
10470 return Register();
10471 if ((Subtarget->hasNDD())) {
10472 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10473 }
10474 if ((!Subtarget->hasNDD())) {
10475 return fastEmitInst_rr(MachineInstOpcode: X86::ADD16rr, RC: &X86::GR16RegClass, Op0, Op1);
10476 }
10477 return Register();
10478}
10479
10480Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10481 if (RetVT.SimpleTy != MVT::i32)
10482 return Register();
10483 if ((Subtarget->hasNDD())) {
10484 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10485 }
10486 if ((!Subtarget->hasNDD())) {
10487 return fastEmitInst_rr(MachineInstOpcode: X86::ADD32rr, RC: &X86::GR32RegClass, Op0, Op1);
10488 }
10489 return Register();
10490}
10491
10492Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10493 if (RetVT.SimpleTy != MVT::i64)
10494 return Register();
10495 if ((Subtarget->hasNDD())) {
10496 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10497 }
10498 if ((!Subtarget->hasNDD())) {
10499 return fastEmitInst_rr(MachineInstOpcode: X86::ADD64rr, RC: &X86::GR64RegClass, Op0, Op1);
10500 }
10501 return Register();
10502}
10503
10504Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10505 if (RetVT.SimpleTy != MVT::v16i8)
10506 return Register();
10507 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10508 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10509 }
10510 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10511 return fastEmitInst_rr(MachineInstOpcode: X86::PADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
10512 }
10513 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10514 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBrr, RC: &X86::VR128RegClass, Op0, Op1);
10515 }
10516 return Register();
10517}
10518
10519Register fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10520 if (RetVT.SimpleTy != MVT::v32i8)
10521 return Register();
10522 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10523 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10524 }
10525 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10526 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10527 }
10528 return Register();
10529}
10530
10531Register fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10532 if (RetVT.SimpleTy != MVT::v64i8)
10533 return Register();
10534 if ((Subtarget->hasBWI())) {
10535 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10536 }
10537 return Register();
10538}
10539
10540Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10541 if (RetVT.SimpleTy != MVT::v8i16)
10542 return Register();
10543 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10544 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10545 }
10546 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10547 return fastEmitInst_rr(MachineInstOpcode: X86::PADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
10548 }
10549 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10550 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
10551 }
10552 return Register();
10553}
10554
10555Register fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10556 if (RetVT.SimpleTy != MVT::v16i16)
10557 return Register();
10558 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10559 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10560 }
10561 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10562 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10563 }
10564 return Register();
10565}
10566
10567Register fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10568 if (RetVT.SimpleTy != MVT::v32i16)
10569 return Register();
10570 if ((Subtarget->hasBWI())) {
10571 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
10572 }
10573 return Register();
10574}
10575
10576Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10577 if (RetVT.SimpleTy != MVT::v4i32)
10578 return Register();
10579 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10580 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10581 }
10582 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10583 return fastEmitInst_rr(MachineInstOpcode: X86::PADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
10584 }
10585 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10586 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
10587 }
10588 return Register();
10589}
10590
10591Register fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
10592 if (RetVT.SimpleTy != MVT::v8i32)
10593 return Register();
10594 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10595 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10596 }
10597 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10598 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10599 }
10600 return Register();
10601}
10602
10603Register fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
10604 if (RetVT.SimpleTy != MVT::v16i32)
10605 return Register();
10606 if ((Subtarget->hasAVX512())) {
10607 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10608 }
10609 return Register();
10610}
10611
10612Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10613 if (RetVT.SimpleTy != MVT::v2i64)
10614 return Register();
10615 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10616 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10617 }
10618 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10619 return fastEmitInst_rr(MachineInstOpcode: X86::PADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
10620 }
10621 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10622 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQrr, RC: &X86::VR128RegClass, Op0, Op1);
10623 }
10624 return Register();
10625}
10626
10627Register fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
10628 if (RetVT.SimpleTy != MVT::v4i64)
10629 return Register();
10630 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10631 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10632 }
10633 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10634 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
10635 }
10636 return Register();
10637}
10638
10639Register fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
10640 if (RetVT.SimpleTy != MVT::v8i64)
10641 return Register();
10642 if ((Subtarget->hasAVX512())) {
10643 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10644 }
10645 return Register();
10646}
10647
10648Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10649 switch (VT.SimpleTy) {
10650 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op1);
10651 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op1);
10652 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10653 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10654 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10655 case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op1);
10656 case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op1);
10657 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10658 case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op1);
10659 case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op1);
10660 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10661 case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op1);
10662 case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op1);
10663 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10664 case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op1);
10665 case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op1);
10666 default: return Register();
10667 }
10668}
10669
10670// FastEmit functions for ISD::AND.
10671
10672Register fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
10673 if (RetVT.SimpleTy != MVT::i8)
10674 return Register();
10675 if ((Subtarget->hasNDD())) {
10676 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
10677 }
10678 if ((!Subtarget->hasNDD())) {
10679 return fastEmitInst_rr(MachineInstOpcode: X86::AND8rr, RC: &X86::GR8RegClass, Op0, Op1);
10680 }
10681 return Register();
10682}
10683
10684Register fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
10685 if (RetVT.SimpleTy != MVT::i16)
10686 return Register();
10687 if ((Subtarget->hasNDD())) {
10688 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
10689 }
10690 if ((!Subtarget->hasNDD())) {
10691 return fastEmitInst_rr(MachineInstOpcode: X86::AND16rr, RC: &X86::GR16RegClass, Op0, Op1);
10692 }
10693 return Register();
10694}
10695
10696Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10697 if (RetVT.SimpleTy != MVT::i32)
10698 return Register();
10699 if ((Subtarget->hasNDD())) {
10700 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
10701 }
10702 if ((!Subtarget->hasNDD())) {
10703 return fastEmitInst_rr(MachineInstOpcode: X86::AND32rr, RC: &X86::GR32RegClass, Op0, Op1);
10704 }
10705 return Register();
10706}
10707
10708Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10709 if (RetVT.SimpleTy != MVT::i64)
10710 return Register();
10711 if ((Subtarget->hasNDD())) {
10712 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
10713 }
10714 if ((!Subtarget->hasNDD())) {
10715 return fastEmitInst_rr(MachineInstOpcode: X86::AND64rr, RC: &X86::GR64RegClass, Op0, Op1);
10716 }
10717 return Register();
10718}
10719
10720Register fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10721 if (RetVT.SimpleTy != MVT::v8i1)
10722 return Register();
10723 if ((Subtarget->hasDQI())) {
10724 return fastEmitInst_rr(MachineInstOpcode: X86::KANDBkk, RC: &X86::VK8RegClass, Op0, Op1);
10725 }
10726 return Register();
10727}
10728
10729Register fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
10730 if (RetVT.SimpleTy != MVT::v16i1)
10731 return Register();
10732 if ((Subtarget->hasAVX512())) {
10733 return fastEmitInst_rr(MachineInstOpcode: X86::KANDWkk, RC: &X86::VK16RegClass, Op0, Op1);
10734 }
10735 return Register();
10736}
10737
10738Register fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
10739 if (RetVT.SimpleTy != MVT::v32i1)
10740 return Register();
10741 if ((Subtarget->hasBWI())) {
10742 return fastEmitInst_rr(MachineInstOpcode: X86::KANDDkk, RC: &X86::VK32RegClass, Op0, Op1);
10743 }
10744 return Register();
10745}
10746
10747Register fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
10748 if (RetVT.SimpleTy != MVT::v64i1)
10749 return Register();
10750 if ((Subtarget->hasBWI())) {
10751 return fastEmitInst_rr(MachineInstOpcode: X86::KANDQkk, RC: &X86::VK64RegClass, Op0, Op1);
10752 }
10753 return Register();
10754}
10755
10756Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10757 if (RetVT.SimpleTy != MVT::v16i8)
10758 return Register();
10759 if ((Subtarget->hasVLX())) {
10760 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10761 }
10762 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10763 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10764 }
10765 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10766 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10767 }
10768 return Register();
10769}
10770
10771Register fastEmit_ISD_AND_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10772 if (RetVT.SimpleTy != MVT::v32i8)
10773 return Register();
10774 if ((Subtarget->hasVLX())) {
10775 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10776 }
10777 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10778 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10779 }
10780 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10781 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10782 }
10783 return Register();
10784}
10785
10786Register fastEmit_ISD_AND_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10787 if (RetVT.SimpleTy != MVT::v64i8)
10788 return Register();
10789 if ((Subtarget->hasAVX512())) {
10790 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10791 }
10792 return Register();
10793}
10794
10795Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10796 if (RetVT.SimpleTy != MVT::v8i16)
10797 return Register();
10798 if ((Subtarget->hasVLX())) {
10799 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10800 }
10801 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10802 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10803 }
10804 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10805 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10806 }
10807 return Register();
10808}
10809
10810Register fastEmit_ISD_AND_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10811 if (RetVT.SimpleTy != MVT::v16i16)
10812 return Register();
10813 if ((Subtarget->hasVLX())) {
10814 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10815 }
10816 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10817 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10818 }
10819 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10820 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10821 }
10822 return Register();
10823}
10824
10825Register fastEmit_ISD_AND_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
10826 if (RetVT.SimpleTy != MVT::v32i16)
10827 return Register();
10828 if ((Subtarget->hasAVX512())) {
10829 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10830 }
10831 return Register();
10832}
10833
10834Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10835 if (RetVT.SimpleTy != MVT::v4i32)
10836 return Register();
10837 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10838 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10839 }
10840 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10841 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10842 }
10843 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10844 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10845 }
10846 return Register();
10847}
10848
10849Register fastEmit_ISD_AND_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
10850 if (RetVT.SimpleTy != MVT::v8i32)
10851 return Register();
10852 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10853 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10854 }
10855 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10856 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10857 }
10858 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10859 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10860 }
10861 return Register();
10862}
10863
10864Register fastEmit_ISD_AND_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
10865 if (RetVT.SimpleTy != MVT::v16i32)
10866 return Register();
10867 if ((Subtarget->hasAVX512())) {
10868 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDDZrr, RC: &X86::VR512RegClass, Op0, Op1);
10869 }
10870 return Register();
10871}
10872
10873Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10874 if (RetVT.SimpleTy != MVT::v2i64)
10875 return Register();
10876 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10877 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10878 }
10879 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10880 return fastEmitInst_rr(MachineInstOpcode: X86::PANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10881 }
10882 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10883 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDrr, RC: &X86::VR128RegClass, Op0, Op1);
10884 }
10885 return Register();
10886}
10887
10888Register fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
10889 if (RetVT.SimpleTy != MVT::v4i64)
10890 return Register();
10891 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10892 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
10893 }
10894 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10895 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10896 }
10897 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10898 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDYrr, RC: &X86::VR256RegClass, Op0, Op1);
10899 }
10900 return Register();
10901}
10902
10903Register fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
10904 if (RetVT.SimpleTy != MVT::v8i64)
10905 return Register();
10906 if ((Subtarget->hasAVX512())) {
10907 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
10908 }
10909 return Register();
10910}
10911
10912Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10913 switch (VT.SimpleTy) {
10914 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op1);
10915 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op1);
10916 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10917 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10918 case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op1);
10919 case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op1);
10920 case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op1);
10921 case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op1);
10922 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10923 case MVT::v32i8: return fastEmit_ISD_AND_MVT_v32i8_rr(RetVT, Op0, Op1);
10924 case MVT::v64i8: return fastEmit_ISD_AND_MVT_v64i8_rr(RetVT, Op0, Op1);
10925 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10926 case MVT::v16i16: return fastEmit_ISD_AND_MVT_v16i16_rr(RetVT, Op0, Op1);
10927 case MVT::v32i16: return fastEmit_ISD_AND_MVT_v32i16_rr(RetVT, Op0, Op1);
10928 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10929 case MVT::v8i32: return fastEmit_ISD_AND_MVT_v8i32_rr(RetVT, Op0, Op1);
10930 case MVT::v16i32: return fastEmit_ISD_AND_MVT_v16i32_rr(RetVT, Op0, Op1);
10931 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10932 case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op1);
10933 case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op1);
10934 default: return Register();
10935 }
10936}
10937
10938// FastEmit functions for ISD::AVGCEILU.
10939
10940Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10941 if (RetVT.SimpleTy != MVT::v16i8)
10942 return Register();
10943 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10944 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10945 }
10946 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10947 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
10948 }
10949 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10950 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBrr, RC: &X86::VR128RegClass, Op0, Op1);
10951 }
10952 return Register();
10953}
10954
10955Register fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
10956 if (RetVT.SimpleTy != MVT::v32i8)
10957 return Register();
10958 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10959 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10960 }
10961 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10962 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBYrr, RC: &X86::VR256RegClass, Op0, Op1);
10963 }
10964 return Register();
10965}
10966
10967Register fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
10968 if (RetVT.SimpleTy != MVT::v64i8)
10969 return Register();
10970 if ((Subtarget->hasBWI())) {
10971 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGBZrr, RC: &X86::VR512RegClass, Op0, Op1);
10972 }
10973 return Register();
10974}
10975
10976Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10977 if (RetVT.SimpleTy != MVT::v8i16)
10978 return Register();
10979 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10980 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
10981 }
10982 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10983 return fastEmitInst_rr(MachineInstOpcode: X86::PAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
10984 }
10985 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10986 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWrr, RC: &X86::VR128RegClass, Op0, Op1);
10987 }
10988 return Register();
10989}
10990
10991Register fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
10992 if (RetVT.SimpleTy != MVT::v16i16)
10993 return Register();
10994 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10995 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
10996 }
10997 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10998 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWYrr, RC: &X86::VR256RegClass, Op0, Op1);
10999 }
11000 return Register();
11001}
11002
11003Register fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
11004 if (RetVT.SimpleTy != MVT::v32i16)
11005 return Register();
11006 if ((Subtarget->hasBWI())) {
11007 return fastEmitInst_rr(MachineInstOpcode: X86::VPAVGWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11008 }
11009 return Register();
11010}
11011
11012Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11013 switch (VT.SimpleTy) {
11014 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
11015 case MVT::v32i8: return fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(RetVT, Op0, Op1);
11016 case MVT::v64i8: return fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(RetVT, Op0, Op1);
11017 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
11018 case MVT::v16i16: return fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(RetVT, Op0, Op1);
11019 case MVT::v32i16: return fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(RetVT, Op0, Op1);
11020 default: return Register();
11021 }
11022}
11023
11024// FastEmit functions for ISD::FADD.
11025
11026Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11027 if (RetVT.SimpleTy != MVT::f16)
11028 return Register();
11029 if ((Subtarget->hasFP16())) {
11030 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11031 }
11032 return Register();
11033}
11034
11035Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11036 if (RetVT.SimpleTy != MVT::f32)
11037 return Register();
11038 if ((Subtarget->hasAVX512())) {
11039 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11040 }
11041 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11042 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11043 }
11044 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11045 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11046 }
11047 if ((!Subtarget->hasSSE1())) {
11048 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11049 }
11050 return Register();
11051}
11052
11053Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11054 if (RetVT.SimpleTy != MVT::f64)
11055 return Register();
11056 if ((Subtarget->hasAVX512())) {
11057 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11058 }
11059 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11060 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11061 }
11062 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11063 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11064 }
11065 if ((!Subtarget->hasSSE2())) {
11066 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11067 }
11068 return Register();
11069}
11070
11071Register fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11072 if (RetVT.SimpleTy != MVT::f80)
11073 return Register();
11074 if ((Subtarget->hasX87())) {
11075 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11076 }
11077 return Register();
11078}
11079
11080Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11081 if (RetVT.SimpleTy != MVT::v8f16)
11082 return Register();
11083 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11084 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11085 }
11086 return Register();
11087}
11088
11089Register fastEmit_ISD_FADD_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11090 if (RetVT.SimpleTy != MVT::v16f16)
11091 return Register();
11092 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11093 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11094 }
11095 return Register();
11096}
11097
11098Register fastEmit_ISD_FADD_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11099 if (RetVT.SimpleTy != MVT::v32f16)
11100 return Register();
11101 if ((Subtarget->hasFP16())) {
11102 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11103 }
11104 return Register();
11105}
11106
11107Register fastEmit_ISD_FADD_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11108 if (RetVT.SimpleTy != MVT::v8bf16)
11109 return Register();
11110 if ((Subtarget->hasAVX10_2())) {
11111 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11112 }
11113 return Register();
11114}
11115
11116Register fastEmit_ISD_FADD_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11117 if (RetVT.SimpleTy != MVT::v16bf16)
11118 return Register();
11119 if ((Subtarget->hasAVX10_2())) {
11120 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11121 }
11122 return Register();
11123}
11124
11125Register fastEmit_ISD_FADD_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11126 if (RetVT.SimpleTy != MVT::v32bf16)
11127 return Register();
11128 if ((Subtarget->hasAVX10_2())) {
11129 return fastEmitInst_rr(MachineInstOpcode: X86::VADDBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11130 }
11131 return Register();
11132}
11133
11134Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11135 if (RetVT.SimpleTy != MVT::v4f32)
11136 return Register();
11137 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11138 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11139 }
11140 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11141 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11142 }
11143 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11144 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11145 }
11146 return Register();
11147}
11148
11149Register fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11150 if (RetVT.SimpleTy != MVT::v8f32)
11151 return Register();
11152 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11153 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11154 }
11155 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11156 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11157 }
11158 return Register();
11159}
11160
11161Register fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11162 if (RetVT.SimpleTy != MVT::v16f32)
11163 return Register();
11164 if ((Subtarget->hasAVX512())) {
11165 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11166 }
11167 return Register();
11168}
11169
11170Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11171 if (RetVT.SimpleTy != MVT::v2f64)
11172 return Register();
11173 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11174 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11175 }
11176 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11177 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11178 }
11179 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11180 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11181 }
11182 return Register();
11183}
11184
11185Register fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11186 if (RetVT.SimpleTy != MVT::v4f64)
11187 return Register();
11188 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11189 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11190 }
11191 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11192 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11193 }
11194 return Register();
11195}
11196
11197Register fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11198 if (RetVT.SimpleTy != MVT::v8f64)
11199 return Register();
11200 if ((Subtarget->hasAVX512())) {
11201 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11202 }
11203 return Register();
11204}
11205
11206Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11207 switch (VT.SimpleTy) {
11208 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11209 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11210 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11211 case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op1);
11212 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11213 case MVT::v16f16: return fastEmit_ISD_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
11214 case MVT::v32f16: return fastEmit_ISD_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
11215 case MVT::v8bf16: return fastEmit_ISD_FADD_MVT_v8bf16_rr(RetVT, Op0, Op1);
11216 case MVT::v16bf16: return fastEmit_ISD_FADD_MVT_v16bf16_rr(RetVT, Op0, Op1);
11217 case MVT::v32bf16: return fastEmit_ISD_FADD_MVT_v32bf16_rr(RetVT, Op0, Op1);
11218 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11219 case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
11220 case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
11221 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11222 case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
11223 case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
11224 default: return Register();
11225 }
11226}
11227
11228// FastEmit functions for ISD::FDIV.
11229
11230Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11231 if (RetVT.SimpleTy != MVT::f16)
11232 return Register();
11233 if ((Subtarget->hasFP16())) {
11234 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11235 }
11236 return Register();
11237}
11238
11239Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11240 if (RetVT.SimpleTy != MVT::f32)
11241 return Register();
11242 if ((Subtarget->hasAVX512())) {
11243 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11244 }
11245 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11246 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11247 }
11248 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11249 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11250 }
11251 if ((!Subtarget->hasSSE1())) {
11252 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11253 }
11254 return Register();
11255}
11256
11257Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11258 if (RetVT.SimpleTy != MVT::f64)
11259 return Register();
11260 if ((Subtarget->hasAVX512())) {
11261 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11262 }
11263 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11264 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11265 }
11266 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11267 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11268 }
11269 if ((!Subtarget->hasSSE2())) {
11270 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11271 }
11272 return Register();
11273}
11274
11275Register fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11276 if (RetVT.SimpleTy != MVT::f80)
11277 return Register();
11278 if ((Subtarget->hasX87())) {
11279 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11280 }
11281 return Register();
11282}
11283
11284Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11285 if (RetVT.SimpleTy != MVT::v8f16)
11286 return Register();
11287 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11288 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11289 }
11290 return Register();
11291}
11292
11293Register fastEmit_ISD_FDIV_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11294 if (RetVT.SimpleTy != MVT::v16f16)
11295 return Register();
11296 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11297 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11298 }
11299 return Register();
11300}
11301
11302Register fastEmit_ISD_FDIV_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11303 if (RetVT.SimpleTy != MVT::v32f16)
11304 return Register();
11305 if ((Subtarget->hasFP16())) {
11306 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11307 }
11308 return Register();
11309}
11310
11311Register fastEmit_ISD_FDIV_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11312 if (RetVT.SimpleTy != MVT::v8bf16)
11313 return Register();
11314 if ((Subtarget->hasAVX10_2())) {
11315 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11316 }
11317 return Register();
11318}
11319
11320Register fastEmit_ISD_FDIV_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11321 if (RetVT.SimpleTy != MVT::v16bf16)
11322 return Register();
11323 if ((Subtarget->hasAVX10_2())) {
11324 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11325 }
11326 return Register();
11327}
11328
11329Register fastEmit_ISD_FDIV_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11330 if (RetVT.SimpleTy != MVT::v32bf16)
11331 return Register();
11332 if ((Subtarget->hasAVX10_2())) {
11333 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11334 }
11335 return Register();
11336}
11337
11338Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11339 if (RetVT.SimpleTy != MVT::v4f32)
11340 return Register();
11341 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11342 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11343 }
11344 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11345 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11346 }
11347 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11348 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11349 }
11350 return Register();
11351}
11352
11353Register fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11354 if (RetVT.SimpleTy != MVT::v8f32)
11355 return Register();
11356 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11357 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11358 }
11359 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11360 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11361 }
11362 return Register();
11363}
11364
11365Register fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11366 if (RetVT.SimpleTy != MVT::v16f32)
11367 return Register();
11368 if ((Subtarget->hasAVX512())) {
11369 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11370 }
11371 return Register();
11372}
11373
11374Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11375 if (RetVT.SimpleTy != MVT::v2f64)
11376 return Register();
11377 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11378 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11379 }
11380 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11381 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11382 }
11383 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11384 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11385 }
11386 return Register();
11387}
11388
11389Register fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11390 if (RetVT.SimpleTy != MVT::v4f64)
11391 return Register();
11392 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11393 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11394 }
11395 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11396 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11397 }
11398 return Register();
11399}
11400
11401Register fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11402 if (RetVT.SimpleTy != MVT::v8f64)
11403 return Register();
11404 if ((Subtarget->hasAVX512())) {
11405 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11406 }
11407 return Register();
11408}
11409
11410Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11411 switch (VT.SimpleTy) {
11412 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11413 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11414 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11415 case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
11416 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11417 case MVT::v16f16: return fastEmit_ISD_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
11418 case MVT::v32f16: return fastEmit_ISD_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
11419 case MVT::v8bf16: return fastEmit_ISD_FDIV_MVT_v8bf16_rr(RetVT, Op0, Op1);
11420 case MVT::v16bf16: return fastEmit_ISD_FDIV_MVT_v16bf16_rr(RetVT, Op0, Op1);
11421 case MVT::v32bf16: return fastEmit_ISD_FDIV_MVT_v32bf16_rr(RetVT, Op0, Op1);
11422 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11423 case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
11424 case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
11425 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11426 case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
11427 case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
11428 default: return Register();
11429 }
11430}
11431
11432// FastEmit functions for ISD::FMUL.
11433
11434Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11435 if (RetVT.SimpleTy != MVT::f16)
11436 return Register();
11437 if ((Subtarget->hasFP16())) {
11438 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11439 }
11440 return Register();
11441}
11442
11443Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11444 if (RetVT.SimpleTy != MVT::f32)
11445 return Register();
11446 if ((Subtarget->hasAVX512())) {
11447 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11448 }
11449 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11450 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11451 }
11452 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11453 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11454 }
11455 if ((!Subtarget->hasSSE1())) {
11456 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11457 }
11458 return Register();
11459}
11460
11461Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11462 if (RetVT.SimpleTy != MVT::f64)
11463 return Register();
11464 if ((Subtarget->hasAVX512())) {
11465 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11466 }
11467 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11468 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11469 }
11470 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11471 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11472 }
11473 if ((!Subtarget->hasSSE2())) {
11474 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11475 }
11476 return Register();
11477}
11478
11479Register fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11480 if (RetVT.SimpleTy != MVT::f80)
11481 return Register();
11482 if ((Subtarget->hasX87())) {
11483 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11484 }
11485 return Register();
11486}
11487
11488Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11489 if (RetVT.SimpleTy != MVT::v8f16)
11490 return Register();
11491 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11492 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11493 }
11494 return Register();
11495}
11496
11497Register fastEmit_ISD_FMUL_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11498 if (RetVT.SimpleTy != MVT::v16f16)
11499 return Register();
11500 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11501 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11502 }
11503 return Register();
11504}
11505
11506Register fastEmit_ISD_FMUL_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11507 if (RetVT.SimpleTy != MVT::v32f16)
11508 return Register();
11509 if ((Subtarget->hasFP16())) {
11510 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11511 }
11512 return Register();
11513}
11514
11515Register fastEmit_ISD_FMUL_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11516 if (RetVT.SimpleTy != MVT::v8bf16)
11517 return Register();
11518 if ((Subtarget->hasAVX10_2())) {
11519 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11520 }
11521 return Register();
11522}
11523
11524Register fastEmit_ISD_FMUL_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11525 if (RetVT.SimpleTy != MVT::v16bf16)
11526 return Register();
11527 if ((Subtarget->hasAVX10_2())) {
11528 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11529 }
11530 return Register();
11531}
11532
11533Register fastEmit_ISD_FMUL_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11534 if (RetVT.SimpleTy != MVT::v32bf16)
11535 return Register();
11536 if ((Subtarget->hasAVX10_2())) {
11537 return fastEmitInst_rr(MachineInstOpcode: X86::VMULBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11538 }
11539 return Register();
11540}
11541
11542Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11543 if (RetVT.SimpleTy != MVT::v4f32)
11544 return Register();
11545 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11546 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11547 }
11548 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11549 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11550 }
11551 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11552 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11553 }
11554 return Register();
11555}
11556
11557Register fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11558 if (RetVT.SimpleTy != MVT::v8f32)
11559 return Register();
11560 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11561 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11562 }
11563 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11564 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11565 }
11566 return Register();
11567}
11568
11569Register fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11570 if (RetVT.SimpleTy != MVT::v16f32)
11571 return Register();
11572 if ((Subtarget->hasAVX512())) {
11573 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11574 }
11575 return Register();
11576}
11577
11578Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11579 if (RetVT.SimpleTy != MVT::v2f64)
11580 return Register();
11581 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11582 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11583 }
11584 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11585 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11586 }
11587 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11588 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11589 }
11590 return Register();
11591}
11592
11593Register fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11594 if (RetVT.SimpleTy != MVT::v4f64)
11595 return Register();
11596 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11597 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11598 }
11599 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11600 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11601 }
11602 return Register();
11603}
11604
11605Register fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11606 if (RetVT.SimpleTy != MVT::v8f64)
11607 return Register();
11608 if ((Subtarget->hasAVX512())) {
11609 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11610 }
11611 return Register();
11612}
11613
11614Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11615 switch (VT.SimpleTy) {
11616 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11617 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11618 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11619 case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
11620 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11621 case MVT::v16f16: return fastEmit_ISD_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
11622 case MVT::v32f16: return fastEmit_ISD_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
11623 case MVT::v8bf16: return fastEmit_ISD_FMUL_MVT_v8bf16_rr(RetVT, Op0, Op1);
11624 case MVT::v16bf16: return fastEmit_ISD_FMUL_MVT_v16bf16_rr(RetVT, Op0, Op1);
11625 case MVT::v32bf16: return fastEmit_ISD_FMUL_MVT_v32bf16_rr(RetVT, Op0, Op1);
11626 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11627 case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
11628 case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
11629 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11630 case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
11631 case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
11632 default: return Register();
11633 }
11634}
11635
11636// FastEmit functions for ISD::FSUB.
11637
11638Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11639 if (RetVT.SimpleTy != MVT::f16)
11640 return Register();
11641 if ((Subtarget->hasFP16())) {
11642 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
11643 }
11644 return Register();
11645}
11646
11647Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11648 if (RetVT.SimpleTy != MVT::f32)
11649 return Register();
11650 if ((Subtarget->hasAVX512())) {
11651 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
11652 }
11653 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11654 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11655 }
11656 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11657 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
11658 }
11659 if ((!Subtarget->hasSSE1())) {
11660 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
11661 }
11662 return Register();
11663}
11664
11665Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11666 if (RetVT.SimpleTy != MVT::f64)
11667 return Register();
11668 if ((Subtarget->hasAVX512())) {
11669 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
11670 }
11671 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11672 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11673 }
11674 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11675 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
11676 }
11677 if ((!Subtarget->hasSSE2())) {
11678 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
11679 }
11680 return Register();
11681}
11682
11683Register fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
11684 if (RetVT.SimpleTy != MVT::f80)
11685 return Register();
11686 if ((Subtarget->hasX87())) {
11687 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
11688 }
11689 return Register();
11690}
11691
11692Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11693 if (RetVT.SimpleTy != MVT::v8f16)
11694 return Register();
11695 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11696 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11697 }
11698 return Register();
11699}
11700
11701Register fastEmit_ISD_FSUB_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
11702 if (RetVT.SimpleTy != MVT::v16f16)
11703 return Register();
11704 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11705 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11706 }
11707 return Register();
11708}
11709
11710Register fastEmit_ISD_FSUB_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
11711 if (RetVT.SimpleTy != MVT::v32f16)
11712 return Register();
11713 if ((Subtarget->hasFP16())) {
11714 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
11715 }
11716 return Register();
11717}
11718
11719Register fastEmit_ISD_FSUB_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11720 if (RetVT.SimpleTy != MVT::v8bf16)
11721 return Register();
11722 if ((Subtarget->hasAVX10_2())) {
11723 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11724 }
11725 return Register();
11726}
11727
11728Register fastEmit_ISD_FSUB_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11729 if (RetVT.SimpleTy != MVT::v16bf16)
11730 return Register();
11731 if ((Subtarget->hasAVX10_2())) {
11732 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11733 }
11734 return Register();
11735}
11736
11737Register fastEmit_ISD_FSUB_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11738 if (RetVT.SimpleTy != MVT::v32bf16)
11739 return Register();
11740 if ((Subtarget->hasAVX10_2())) {
11741 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
11742 }
11743 return Register();
11744}
11745
11746Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11747 if (RetVT.SimpleTy != MVT::v4f32)
11748 return Register();
11749 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11750 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11751 }
11752 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11753 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11754 }
11755 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11756 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
11757 }
11758 return Register();
11759}
11760
11761Register fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
11762 if (RetVT.SimpleTy != MVT::v8f32)
11763 return Register();
11764 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11765 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11766 }
11767 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11768 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
11769 }
11770 return Register();
11771}
11772
11773Register fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
11774 if (RetVT.SimpleTy != MVT::v16f32)
11775 return Register();
11776 if ((Subtarget->hasAVX512())) {
11777 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
11778 }
11779 return Register();
11780}
11781
11782Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11783 if (RetVT.SimpleTy != MVT::v2f64)
11784 return Register();
11785 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11786 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11787 }
11788 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11789 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11790 }
11791 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11792 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
11793 }
11794 return Register();
11795}
11796
11797Register fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
11798 if (RetVT.SimpleTy != MVT::v4f64)
11799 return Register();
11800 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11801 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11802 }
11803 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11804 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11805 }
11806 return Register();
11807}
11808
11809Register fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
11810 if (RetVT.SimpleTy != MVT::v8f64)
11811 return Register();
11812 if ((Subtarget->hasAVX512())) {
11813 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11814 }
11815 return Register();
11816}
11817
11818Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11819 switch (VT.SimpleTy) {
11820 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11821 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11822 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11823 case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
11824 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11825 case MVT::v16f16: return fastEmit_ISD_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
11826 case MVT::v32f16: return fastEmit_ISD_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
11827 case MVT::v8bf16: return fastEmit_ISD_FSUB_MVT_v8bf16_rr(RetVT, Op0, Op1);
11828 case MVT::v16bf16: return fastEmit_ISD_FSUB_MVT_v16bf16_rr(RetVT, Op0, Op1);
11829 case MVT::v32bf16: return fastEmit_ISD_FSUB_MVT_v32bf16_rr(RetVT, Op0, Op1);
11830 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11831 case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
11832 case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
11833 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11834 case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
11835 case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
11836 default: return Register();
11837 }
11838}
11839
11840// FastEmit functions for ISD::MUL.
11841
11842Register fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
11843 if (RetVT.SimpleTy != MVT::i8)
11844 return Register();
11845 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::AL).addReg(RegNo: Op0);
11846 return fastEmitInst_r(MachineInstOpcode: X86::MUL8r, RC: &X86::GR8RegClass, Op0: Op1);
11847}
11848
11849Register fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
11850 if (RetVT.SimpleTy != MVT::i16)
11851 return Register();
11852 if ((Subtarget->hasNDD())) {
11853 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
11854 }
11855 if ((!Subtarget->hasNDD())) {
11856 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL16rr, RC: &X86::GR16RegClass, Op0, Op1);
11857 }
11858 return Register();
11859}
11860
11861Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11862 if (RetVT.SimpleTy != MVT::i32)
11863 return Register();
11864 if ((Subtarget->hasNDD())) {
11865 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
11866 }
11867 if ((!Subtarget->hasNDD())) {
11868 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL32rr, RC: &X86::GR32RegClass, Op0, Op1);
11869 }
11870 return Register();
11871}
11872
11873Register fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11874 if (RetVT.SimpleTy != MVT::i64)
11875 return Register();
11876 if ((Subtarget->hasNDD())) {
11877 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
11878 }
11879 if ((!Subtarget->hasNDD())) {
11880 return fastEmitInst_rr(MachineInstOpcode: X86::IMUL64rr, RC: &X86::GR64RegClass, Op0, Op1);
11881 }
11882 return Register();
11883}
11884
11885Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11886 if (RetVT.SimpleTy != MVT::v8i16)
11887 return Register();
11888 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11889 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11890 }
11891 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11892 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
11893 }
11894 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11895 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWrr, RC: &X86::VR128RegClass, Op0, Op1);
11896 }
11897 return Register();
11898}
11899
11900Register fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
11901 if (RetVT.SimpleTy != MVT::v16i16)
11902 return Register();
11903 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11904 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11905 }
11906 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11907 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWYrr, RC: &X86::VR256RegClass, Op0, Op1);
11908 }
11909 return Register();
11910}
11911
11912Register fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
11913 if (RetVT.SimpleTy != MVT::v32i16)
11914 return Register();
11915 if ((Subtarget->hasBWI())) {
11916 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLWZrr, RC: &X86::VR512RegClass, Op0, Op1);
11917 }
11918 return Register();
11919}
11920
11921Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11922 if (RetVT.SimpleTy != MVT::v4i32)
11923 return Register();
11924 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11925 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11926 }
11927 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
11928 return fastEmitInst_rr(MachineInstOpcode: X86::PMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
11929 }
11930 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11931 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDrr, RC: &X86::VR128RegClass, Op0, Op1);
11932 }
11933 return Register();
11934}
11935
11936Register fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
11937 if (RetVT.SimpleTy != MVT::v8i32)
11938 return Register();
11939 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11940 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11941 }
11942 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11943 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDYrr, RC: &X86::VR256RegClass, Op0, Op1);
11944 }
11945 return Register();
11946}
11947
11948Register fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
11949 if (RetVT.SimpleTy != MVT::v16i32)
11950 return Register();
11951 if ((Subtarget->hasAVX512())) {
11952 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLDZrr, RC: &X86::VR512RegClass, Op0, Op1);
11953 }
11954 return Register();
11955}
11956
11957Register fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11958 if (RetVT.SimpleTy != MVT::v2i64)
11959 return Register();
11960 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
11961 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
11962 }
11963 return Register();
11964}
11965
11966Register fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
11967 if (RetVT.SimpleTy != MVT::v4i64)
11968 return Register();
11969 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
11970 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
11971 }
11972 return Register();
11973}
11974
11975Register fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
11976 if (RetVT.SimpleTy != MVT::v8i64)
11977 return Register();
11978 if ((Subtarget->hasDQI())) {
11979 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULLQZrr, RC: &X86::VR512RegClass, Op0, Op1);
11980 }
11981 return Register();
11982}
11983
11984Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11985 switch (VT.SimpleTy) {
11986 case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op1);
11987 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op1);
11988 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
11989 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op1);
11990 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
11991 case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op1);
11992 case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op1);
11993 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
11994 case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op1);
11995 case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op1);
11996 case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op1);
11997 case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op1);
11998 case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op1);
11999 default: return Register();
12000 }
12001}
12002
12003// FastEmit functions for ISD::MULHS.
12004
12005Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12006 if (RetVT.SimpleTy != MVT::v8i16)
12007 return Register();
12008 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12009 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12010 }
12011 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12012 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
12013 }
12014 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12015 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWrr, RC: &X86::VR128RegClass, Op0, Op1);
12016 }
12017 return Register();
12018}
12019
12020Register fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12021 if (RetVT.SimpleTy != MVT::v16i16)
12022 return Register();
12023 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12024 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12025 }
12026 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12027 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12028 }
12029 return Register();
12030}
12031
12032Register fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12033 if (RetVT.SimpleTy != MVT::v32i16)
12034 return Register();
12035 if ((Subtarget->hasBWI())) {
12036 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12037 }
12038 return Register();
12039}
12040
12041Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12042 switch (VT.SimpleTy) {
12043 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
12044 case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op1);
12045 case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op1);
12046 default: return Register();
12047 }
12048}
12049
12050// FastEmit functions for ISD::MULHU.
12051
12052Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12053 if (RetVT.SimpleTy != MVT::v8i16)
12054 return Register();
12055 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12056 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12057 }
12058 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12059 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12060 }
12061 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12062 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWrr, RC: &X86::VR128RegClass, Op0, Op1);
12063 }
12064 return Register();
12065}
12066
12067Register fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12068 if (RetVT.SimpleTy != MVT::v16i16)
12069 return Register();
12070 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12071 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12072 }
12073 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12074 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
12075 }
12076 return Register();
12077}
12078
12079Register fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12080 if (RetVT.SimpleTy != MVT::v32i16)
12081 return Register();
12082 if ((Subtarget->hasBWI())) {
12083 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
12084 }
12085 return Register();
12086}
12087
12088Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12089 switch (VT.SimpleTy) {
12090 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
12091 case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op1);
12092 case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op1);
12093 default: return Register();
12094 }
12095}
12096
12097// FastEmit functions for ISD::OR.
12098
12099Register fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12100 if (RetVT.SimpleTy != MVT::i8)
12101 return Register();
12102 if ((Subtarget->hasNDD())) {
12103 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
12104 }
12105 if ((!Subtarget->hasNDD())) {
12106 return fastEmitInst_rr(MachineInstOpcode: X86::OR8rr, RC: &X86::GR8RegClass, Op0, Op1);
12107 }
12108 return Register();
12109}
12110
12111Register fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
12112 if (RetVT.SimpleTy != MVT::i16)
12113 return Register();
12114 if ((Subtarget->hasNDD())) {
12115 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
12116 }
12117 if ((!Subtarget->hasNDD())) {
12118 return fastEmitInst_rr(MachineInstOpcode: X86::OR16rr, RC: &X86::GR16RegClass, Op0, Op1);
12119 }
12120 return Register();
12121}
12122
12123Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12124 if (RetVT.SimpleTy != MVT::i32)
12125 return Register();
12126 if ((Subtarget->hasNDD())) {
12127 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
12128 }
12129 if ((!Subtarget->hasNDD())) {
12130 return fastEmitInst_rr(MachineInstOpcode: X86::OR32rr, RC: &X86::GR32RegClass, Op0, Op1);
12131 }
12132 return Register();
12133}
12134
12135Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12136 if (RetVT.SimpleTy != MVT::i64)
12137 return Register();
12138 if ((Subtarget->hasNDD())) {
12139 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
12140 }
12141 if ((!Subtarget->hasNDD())) {
12142 return fastEmitInst_rr(MachineInstOpcode: X86::OR64rr, RC: &X86::GR64RegClass, Op0, Op1);
12143 }
12144 return Register();
12145}
12146
12147Register fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
12148 if (RetVT.SimpleTy != MVT::v8i1)
12149 return Register();
12150 if ((Subtarget->hasDQI())) {
12151 return fastEmitInst_rr(MachineInstOpcode: X86::KORBkk, RC: &X86::VK8RegClass, Op0, Op1);
12152 }
12153 return Register();
12154}
12155
12156Register fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
12157 if (RetVT.SimpleTy != MVT::v16i1)
12158 return Register();
12159 if ((Subtarget->hasAVX512())) {
12160 return fastEmitInst_rr(MachineInstOpcode: X86::KORWkk, RC: &X86::VK16RegClass, Op0, Op1);
12161 }
12162 return Register();
12163}
12164
12165Register fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
12166 if (RetVT.SimpleTy != MVT::v32i1)
12167 return Register();
12168 if ((Subtarget->hasBWI())) {
12169 return fastEmitInst_rr(MachineInstOpcode: X86::KORDkk, RC: &X86::VK32RegClass, Op0, Op1);
12170 }
12171 return Register();
12172}
12173
12174Register fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
12175 if (RetVT.SimpleTy != MVT::v64i1)
12176 return Register();
12177 if ((Subtarget->hasBWI())) {
12178 return fastEmitInst_rr(MachineInstOpcode: X86::KORQkk, RC: &X86::VK64RegClass, Op0, Op1);
12179 }
12180 return Register();
12181}
12182
12183Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12184 if (RetVT.SimpleTy != MVT::v16i8)
12185 return Register();
12186 if ((Subtarget->hasVLX())) {
12187 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12188 }
12189 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12190 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12191 }
12192 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12193 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12194 }
12195 return Register();
12196}
12197
12198Register fastEmit_ISD_OR_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
12199 if (RetVT.SimpleTy != MVT::v32i8)
12200 return Register();
12201 if ((Subtarget->hasVLX())) {
12202 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12203 }
12204 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12205 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12206 }
12207 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12208 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12209 }
12210 return Register();
12211}
12212
12213Register fastEmit_ISD_OR_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
12214 if (RetVT.SimpleTy != MVT::v64i8)
12215 return Register();
12216 if ((Subtarget->hasAVX512())) {
12217 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12218 }
12219 return Register();
12220}
12221
12222Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12223 if (RetVT.SimpleTy != MVT::v8i16)
12224 return Register();
12225 if ((Subtarget->hasVLX())) {
12226 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12227 }
12228 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12229 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12230 }
12231 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12232 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12233 }
12234 return Register();
12235}
12236
12237Register fastEmit_ISD_OR_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
12238 if (RetVT.SimpleTy != MVT::v16i16)
12239 return Register();
12240 if ((Subtarget->hasVLX())) {
12241 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12242 }
12243 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12244 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12245 }
12246 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12247 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12248 }
12249 return Register();
12250}
12251
12252Register fastEmit_ISD_OR_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
12253 if (RetVT.SimpleTy != MVT::v32i16)
12254 return Register();
12255 if ((Subtarget->hasAVX512())) {
12256 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12257 }
12258 return Register();
12259}
12260
12261Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12262 if (RetVT.SimpleTy != MVT::v4i32)
12263 return Register();
12264 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12265 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12266 }
12267 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12268 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12269 }
12270 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12271 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12272 }
12273 return Register();
12274}
12275
12276Register fastEmit_ISD_OR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12277 if (RetVT.SimpleTy != MVT::v8i32)
12278 return Register();
12279 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12280 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12281 }
12282 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12283 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12284 }
12285 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12286 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12287 }
12288 return Register();
12289}
12290
12291Register fastEmit_ISD_OR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12292 if (RetVT.SimpleTy != MVT::v16i32)
12293 return Register();
12294 if ((Subtarget->hasAVX512())) {
12295 return fastEmitInst_rr(MachineInstOpcode: X86::VPORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12296 }
12297 return Register();
12298}
12299
12300Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12301 if (RetVT.SimpleTy != MVT::v2i64)
12302 return Register();
12303 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12304 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12305 }
12306 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12307 return fastEmitInst_rr(MachineInstOpcode: X86::PORrr, RC: &X86::VR128RegClass, Op0, Op1);
12308 }
12309 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12310 return fastEmitInst_rr(MachineInstOpcode: X86::VPORrr, RC: &X86::VR128RegClass, Op0, Op1);
12311 }
12312 return Register();
12313}
12314
12315Register fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12316 if (RetVT.SimpleTy != MVT::v4i64)
12317 return Register();
12318 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12319 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12320 }
12321 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12322 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12323 }
12324 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12325 return fastEmitInst_rr(MachineInstOpcode: X86::VPORYrr, RC: &X86::VR256RegClass, Op0, Op1);
12326 }
12327 return Register();
12328}
12329
12330Register fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12331 if (RetVT.SimpleTy != MVT::v8i64)
12332 return Register();
12333 if ((Subtarget->hasAVX512())) {
12334 return fastEmitInst_rr(MachineInstOpcode: X86::VPORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12335 }
12336 return Register();
12337}
12338
12339Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12340 switch (VT.SimpleTy) {
12341 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op1);
12342 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op1);
12343 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12344 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12345 case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op1);
12346 case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op1);
12347 case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op1);
12348 case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op1);
12349 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12350 case MVT::v32i8: return fastEmit_ISD_OR_MVT_v32i8_rr(RetVT, Op0, Op1);
12351 case MVT::v64i8: return fastEmit_ISD_OR_MVT_v64i8_rr(RetVT, Op0, Op1);
12352 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12353 case MVT::v16i16: return fastEmit_ISD_OR_MVT_v16i16_rr(RetVT, Op0, Op1);
12354 case MVT::v32i16: return fastEmit_ISD_OR_MVT_v32i16_rr(RetVT, Op0, Op1);
12355 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12356 case MVT::v8i32: return fastEmit_ISD_OR_MVT_v8i32_rr(RetVT, Op0, Op1);
12357 case MVT::v16i32: return fastEmit_ISD_OR_MVT_v16i32_rr(RetVT, Op0, Op1);
12358 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12359 case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op1);
12360 case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op1);
12361 default: return Register();
12362 }
12363}
12364
12365// FastEmit functions for ISD::PDEP.
12366
12367Register fastEmit_ISD_PDEP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12368 if (RetVT.SimpleTy != MVT::i32)
12369 return Register();
12370 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12371 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
12372 }
12373 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12374 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP32rr, RC: &X86::GR32RegClass, Op0, Op1);
12375 }
12376 return Register();
12377}
12378
12379Register fastEmit_ISD_PDEP_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12380 if (RetVT.SimpleTy != MVT::i64)
12381 return Register();
12382 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12383 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
12384 }
12385 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12386 return fastEmitInst_rr(MachineInstOpcode: X86::PDEP64rr, RC: &X86::GR64RegClass, Op0, Op1);
12387 }
12388 return Register();
12389}
12390
12391Register fastEmit_ISD_PDEP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12392 switch (VT.SimpleTy) {
12393 case MVT::i32: return fastEmit_ISD_PDEP_MVT_i32_rr(RetVT, Op0, Op1);
12394 case MVT::i64: return fastEmit_ISD_PDEP_MVT_i64_rr(RetVT, Op0, Op1);
12395 default: return Register();
12396 }
12397}
12398
12399// FastEmit functions for ISD::PEXT.
12400
12401Register fastEmit_ISD_PEXT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12402 if (RetVT.SimpleTy != MVT::i32)
12403 return Register();
12404 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12405 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
12406 }
12407 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12408 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT32rr, RC: &X86::GR32RegClass, Op0, Op1);
12409 }
12410 return Register();
12411}
12412
12413Register fastEmit_ISD_PEXT_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12414 if (RetVT.SimpleTy != MVT::i64)
12415 return Register();
12416 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
12417 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
12418 }
12419 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
12420 return fastEmitInst_rr(MachineInstOpcode: X86::PEXT64rr, RC: &X86::GR64RegClass, Op0, Op1);
12421 }
12422 return Register();
12423}
12424
12425Register fastEmit_ISD_PEXT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12426 switch (VT.SimpleTy) {
12427 case MVT::i32: return fastEmit_ISD_PEXT_MVT_i32_rr(RetVT, Op0, Op1);
12428 case MVT::i64: return fastEmit_ISD_PEXT_MVT_i64_rr(RetVT, Op0, Op1);
12429 default: return Register();
12430 }
12431}
12432
12433// FastEmit functions for ISD::PSEUDO_FMAX.
12434
12435Register fastEmit_ISD_PSEUDO_FMAX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12436 if (RetVT.SimpleTy != MVT::f16)
12437 return Register();
12438 if ((Subtarget->hasFP16())) {
12439 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
12440 }
12441 return Register();
12442}
12443
12444Register fastEmit_ISD_PSEUDO_FMAX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12445 if (RetVT.SimpleTy != MVT::f32)
12446 return Register();
12447 if ((Subtarget->hasAVX512())) {
12448 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
12449 }
12450 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12451 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
12452 }
12453 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12454 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
12455 }
12456 return Register();
12457}
12458
12459Register fastEmit_ISD_PSEUDO_FMAX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12460 if (RetVT.SimpleTy != MVT::f64)
12461 return Register();
12462 if ((Subtarget->hasAVX512())) {
12463 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
12464 }
12465 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12466 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
12467 }
12468 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12469 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
12470 }
12471 return Register();
12472}
12473
12474Register fastEmit_ISD_PSEUDO_FMAX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12475 if (RetVT.SimpleTy != MVT::v8f16)
12476 return Register();
12477 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
12478 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12479 }
12480 return Register();
12481}
12482
12483Register fastEmit_ISD_PSEUDO_FMAX_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
12484 if (RetVT.SimpleTy != MVT::v16f16)
12485 return Register();
12486 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
12487 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12488 }
12489 return Register();
12490}
12491
12492Register fastEmit_ISD_PSEUDO_FMAX_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
12493 if (RetVT.SimpleTy != MVT::v32f16)
12494 return Register();
12495 if ((Subtarget->hasFP16())) {
12496 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
12497 }
12498 return Register();
12499}
12500
12501Register fastEmit_ISD_PSEUDO_FMAX_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12502 if (RetVT.SimpleTy != MVT::v8bf16)
12503 return Register();
12504 if ((Subtarget->hasAVX10_2())) {
12505 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12506 }
12507 return Register();
12508}
12509
12510Register fastEmit_ISD_PSEUDO_FMAX_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12511 if (RetVT.SimpleTy != MVT::v16bf16)
12512 return Register();
12513 if ((Subtarget->hasAVX10_2())) {
12514 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12515 }
12516 return Register();
12517}
12518
12519Register fastEmit_ISD_PSEUDO_FMAX_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12520 if (RetVT.SimpleTy != MVT::v32bf16)
12521 return Register();
12522 if ((Subtarget->hasAVX10_2())) {
12523 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
12524 }
12525 return Register();
12526}
12527
12528Register fastEmit_ISD_PSEUDO_FMAX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12529 if (RetVT.SimpleTy != MVT::v4f32)
12530 return Register();
12531 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12532 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12533 }
12534 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12535 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
12536 }
12537 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12538 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
12539 }
12540 return Register();
12541}
12542
12543Register fastEmit_ISD_PSEUDO_FMAX_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
12544 if (RetVT.SimpleTy != MVT::v8f32)
12545 return Register();
12546 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12547 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12548 }
12549 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12550 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12551 }
12552 return Register();
12553}
12554
12555Register fastEmit_ISD_PSEUDO_FMAX_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
12556 if (RetVT.SimpleTy != MVT::v16f32)
12557 return Register();
12558 if ((Subtarget->hasAVX512())) {
12559 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
12560 }
12561 return Register();
12562}
12563
12564Register fastEmit_ISD_PSEUDO_FMAX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12565 if (RetVT.SimpleTy != MVT::v2f64)
12566 return Register();
12567 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12568 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12569 }
12570 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12571 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
12572 }
12573 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12574 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
12575 }
12576 return Register();
12577}
12578
12579Register fastEmit_ISD_PSEUDO_FMAX_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
12580 if (RetVT.SimpleTy != MVT::v4f64)
12581 return Register();
12582 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12583 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12584 }
12585 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12586 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12587 }
12588 return Register();
12589}
12590
12591Register fastEmit_ISD_PSEUDO_FMAX_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
12592 if (RetVT.SimpleTy != MVT::v8f64)
12593 return Register();
12594 if ((Subtarget->hasAVX512())) {
12595 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12596 }
12597 return Register();
12598}
12599
12600Register fastEmit_ISD_PSEUDO_FMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12601 switch (VT.SimpleTy) {
12602 case MVT::f16: return fastEmit_ISD_PSEUDO_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
12603 case MVT::f32: return fastEmit_ISD_PSEUDO_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
12604 case MVT::f64: return fastEmit_ISD_PSEUDO_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
12605 case MVT::v8f16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
12606 case MVT::v16f16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
12607 case MVT::v32f16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
12608 case MVT::v8bf16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v8bf16_rr(RetVT, Op0, Op1);
12609 case MVT::v16bf16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v16bf16_rr(RetVT, Op0, Op1);
12610 case MVT::v32bf16: return fastEmit_ISD_PSEUDO_FMAX_MVT_v32bf16_rr(RetVT, Op0, Op1);
12611 case MVT::v4f32: return fastEmit_ISD_PSEUDO_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
12612 case MVT::v8f32: return fastEmit_ISD_PSEUDO_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
12613 case MVT::v16f32: return fastEmit_ISD_PSEUDO_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
12614 case MVT::v2f64: return fastEmit_ISD_PSEUDO_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
12615 case MVT::v4f64: return fastEmit_ISD_PSEUDO_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
12616 case MVT::v8f64: return fastEmit_ISD_PSEUDO_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
12617 default: return Register();
12618 }
12619}
12620
12621// FastEmit functions for ISD::PSEUDO_FMIN.
12622
12623Register fastEmit_ISD_PSEUDO_FMIN_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12624 if (RetVT.SimpleTy != MVT::f16)
12625 return Register();
12626 if ((Subtarget->hasFP16())) {
12627 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
12628 }
12629 return Register();
12630}
12631
12632Register fastEmit_ISD_PSEUDO_FMIN_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12633 if (RetVT.SimpleTy != MVT::f32)
12634 return Register();
12635 if ((Subtarget->hasAVX512())) {
12636 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
12637 }
12638 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12639 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
12640 }
12641 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12642 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
12643 }
12644 return Register();
12645}
12646
12647Register fastEmit_ISD_PSEUDO_FMIN_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12648 if (RetVT.SimpleTy != MVT::f64)
12649 return Register();
12650 if ((Subtarget->hasAVX512())) {
12651 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
12652 }
12653 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12654 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
12655 }
12656 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12657 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
12658 }
12659 return Register();
12660}
12661
12662Register fastEmit_ISD_PSEUDO_FMIN_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12663 if (RetVT.SimpleTy != MVT::v8f16)
12664 return Register();
12665 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
12666 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12667 }
12668 return Register();
12669}
12670
12671Register fastEmit_ISD_PSEUDO_FMIN_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
12672 if (RetVT.SimpleTy != MVT::v16f16)
12673 return Register();
12674 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
12675 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12676 }
12677 return Register();
12678}
12679
12680Register fastEmit_ISD_PSEUDO_FMIN_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
12681 if (RetVT.SimpleTy != MVT::v32f16)
12682 return Register();
12683 if ((Subtarget->hasFP16())) {
12684 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
12685 }
12686 return Register();
12687}
12688
12689Register fastEmit_ISD_PSEUDO_FMIN_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12690 if (RetVT.SimpleTy != MVT::v8bf16)
12691 return Register();
12692 if ((Subtarget->hasAVX10_2())) {
12693 return fastEmitInst_rr(MachineInstOpcode: X86::VMINBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12694 }
12695 return Register();
12696}
12697
12698Register fastEmit_ISD_PSEUDO_FMIN_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12699 if (RetVT.SimpleTy != MVT::v16bf16)
12700 return Register();
12701 if ((Subtarget->hasAVX10_2())) {
12702 return fastEmitInst_rr(MachineInstOpcode: X86::VMINBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12703 }
12704 return Register();
12705}
12706
12707Register fastEmit_ISD_PSEUDO_FMIN_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
12708 if (RetVT.SimpleTy != MVT::v32bf16)
12709 return Register();
12710 if ((Subtarget->hasAVX10_2())) {
12711 return fastEmitInst_rr(MachineInstOpcode: X86::VMINBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
12712 }
12713 return Register();
12714}
12715
12716Register fastEmit_ISD_PSEUDO_FMIN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12717 if (RetVT.SimpleTy != MVT::v4f32)
12718 return Register();
12719 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12720 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12721 }
12722 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12723 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
12724 }
12725 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12726 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
12727 }
12728 return Register();
12729}
12730
12731Register fastEmit_ISD_PSEUDO_FMIN_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
12732 if (RetVT.SimpleTy != MVT::v8f32)
12733 return Register();
12734 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12735 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12736 }
12737 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12738 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
12739 }
12740 return Register();
12741}
12742
12743Register fastEmit_ISD_PSEUDO_FMIN_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
12744 if (RetVT.SimpleTy != MVT::v16f32)
12745 return Register();
12746 if ((Subtarget->hasAVX512())) {
12747 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
12748 }
12749 return Register();
12750}
12751
12752Register fastEmit_ISD_PSEUDO_FMIN_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12753 if (RetVT.SimpleTy != MVT::v2f64)
12754 return Register();
12755 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12756 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12757 }
12758 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12759 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
12760 }
12761 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12762 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
12763 }
12764 return Register();
12765}
12766
12767Register fastEmit_ISD_PSEUDO_FMIN_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
12768 if (RetVT.SimpleTy != MVT::v4f64)
12769 return Register();
12770 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12771 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12772 }
12773 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12774 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
12775 }
12776 return Register();
12777}
12778
12779Register fastEmit_ISD_PSEUDO_FMIN_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
12780 if (RetVT.SimpleTy != MVT::v8f64)
12781 return Register();
12782 if ((Subtarget->hasAVX512())) {
12783 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12784 }
12785 return Register();
12786}
12787
12788Register fastEmit_ISD_PSEUDO_FMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12789 switch (VT.SimpleTy) {
12790 case MVT::f16: return fastEmit_ISD_PSEUDO_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
12791 case MVT::f32: return fastEmit_ISD_PSEUDO_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
12792 case MVT::f64: return fastEmit_ISD_PSEUDO_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
12793 case MVT::v8f16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
12794 case MVT::v16f16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
12795 case MVT::v32f16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
12796 case MVT::v8bf16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v8bf16_rr(RetVT, Op0, Op1);
12797 case MVT::v16bf16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v16bf16_rr(RetVT, Op0, Op1);
12798 case MVT::v32bf16: return fastEmit_ISD_PSEUDO_FMIN_MVT_v32bf16_rr(RetVT, Op0, Op1);
12799 case MVT::v4f32: return fastEmit_ISD_PSEUDO_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
12800 case MVT::v8f32: return fastEmit_ISD_PSEUDO_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
12801 case MVT::v16f32: return fastEmit_ISD_PSEUDO_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
12802 case MVT::v2f64: return fastEmit_ISD_PSEUDO_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
12803 case MVT::v4f64: return fastEmit_ISD_PSEUDO_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
12804 case MVT::v8f64: return fastEmit_ISD_PSEUDO_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
12805 default: return Register();
12806 }
12807}
12808
12809// FastEmit functions for ISD::ROTL.
12810
12811Register fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12812 if (RetVT.SimpleTy != MVT::i8)
12813 return Register();
12814 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12815 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12816 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL_ND, RC: &X86::GR8RegClass, Op0);
12817 }
12818 if ((!Subtarget->hasNDD())) {
12819 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12820 return fastEmitInst_r(MachineInstOpcode: X86::ROL8rCL, RC: &X86::GR8RegClass, Op0);
12821 }
12822 return Register();
12823}
12824
12825Register fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12826 if (RetVT.SimpleTy != MVT::v16i8)
12827 return Register();
12828 if ((Subtarget->hasXOP())) {
12829 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTBrr, RC: &X86::VR128RegClass, Op0, Op1);
12830 }
12831 return Register();
12832}
12833
12834Register fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12835 if (RetVT.SimpleTy != MVT::v8i16)
12836 return Register();
12837 if ((Subtarget->hasXOP())) {
12838 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTWrr, RC: &X86::VR128RegClass, Op0, Op1);
12839 }
12840 return Register();
12841}
12842
12843Register fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12844 if (RetVT.SimpleTy != MVT::v4i32)
12845 return Register();
12846 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12847 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12848 }
12849 if ((Subtarget->hasXOP())) {
12850 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTDrr, RC: &X86::VR128RegClass, Op0, Op1);
12851 }
12852 return Register();
12853}
12854
12855Register fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12856 if (RetVT.SimpleTy != MVT::v8i32)
12857 return Register();
12858 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12859 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12860 }
12861 return Register();
12862}
12863
12864Register fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12865 if (RetVT.SimpleTy != MVT::v16i32)
12866 return Register();
12867 if ((Subtarget->hasAVX512())) {
12868 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12869 }
12870 return Register();
12871}
12872
12873Register fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12874 if (RetVT.SimpleTy != MVT::v2i64)
12875 return Register();
12876 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12877 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12878 }
12879 if ((Subtarget->hasXOP())) {
12880 return fastEmitInst_rr(MachineInstOpcode: X86::VPROTQrr, RC: &X86::VR128RegClass, Op0, Op1);
12881 }
12882 return Register();
12883}
12884
12885Register fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12886 if (RetVT.SimpleTy != MVT::v4i64)
12887 return Register();
12888 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12889 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12890 }
12891 return Register();
12892}
12893
12894Register fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12895 if (RetVT.SimpleTy != MVT::v8i64)
12896 return Register();
12897 if ((Subtarget->hasAVX512())) {
12898 return fastEmitInst_rr(MachineInstOpcode: X86::VPROLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12899 }
12900 return Register();
12901}
12902
12903Register fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12904 switch (VT.SimpleTy) {
12905 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op1);
12906 case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op1);
12907 case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op1);
12908 case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op1);
12909 case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op1);
12910 case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op1);
12911 case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op1);
12912 case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op1);
12913 case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op1);
12914 default: return Register();
12915 }
12916}
12917
12918// FastEmit functions for ISD::ROTR.
12919
12920Register fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
12921 if (RetVT.SimpleTy != MVT::i8)
12922 return Register();
12923 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
12924 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12925 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL_ND, RC: &X86::GR8RegClass, Op0);
12926 }
12927 if ((!Subtarget->hasNDD())) {
12928 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
12929 return fastEmitInst_r(MachineInstOpcode: X86::ROR8rCL, RC: &X86::GR8RegClass, Op0);
12930 }
12931 return Register();
12932}
12933
12934Register fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12935 if (RetVT.SimpleTy != MVT::v4i32)
12936 return Register();
12937 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12938 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12939 }
12940 return Register();
12941}
12942
12943Register fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
12944 if (RetVT.SimpleTy != MVT::v8i32)
12945 return Register();
12946 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12947 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12948 }
12949 return Register();
12950}
12951
12952Register fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
12953 if (RetVT.SimpleTy != MVT::v16i32)
12954 return Register();
12955 if ((Subtarget->hasAVX512())) {
12956 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
12957 }
12958 return Register();
12959}
12960
12961Register fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12962 if (RetVT.SimpleTy != MVT::v2i64)
12963 return Register();
12964 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12965 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
12966 }
12967 return Register();
12968}
12969
12970Register fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
12971 if (RetVT.SimpleTy != MVT::v4i64)
12972 return Register();
12973 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12974 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
12975 }
12976 return Register();
12977}
12978
12979Register fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
12980 if (RetVT.SimpleTy != MVT::v8i64)
12981 return Register();
12982 if ((Subtarget->hasAVX512())) {
12983 return fastEmitInst_rr(MachineInstOpcode: X86::VPRORVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
12984 }
12985 return Register();
12986}
12987
12988Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12989 switch (VT.SimpleTy) {
12990 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op1);
12991 case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op1);
12992 case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op1);
12993 case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op1);
12994 case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op1);
12995 case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op1);
12996 case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op1);
12997 default: return Register();
12998 }
12999}
13000
13001// FastEmit functions for ISD::SADDSAT.
13002
13003Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13004 if (RetVT.SimpleTy != MVT::v16i8)
13005 return Register();
13006 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13007 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13008 }
13009 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13010 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13011 }
13012 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13013 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13014 }
13015 return Register();
13016}
13017
13018Register fastEmit_ISD_SADDSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13019 if (RetVT.SimpleTy != MVT::v32i8)
13020 return Register();
13021 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13022 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13023 }
13024 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13025 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13026 }
13027 return Register();
13028}
13029
13030Register fastEmit_ISD_SADDSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13031 if (RetVT.SimpleTy != MVT::v64i8)
13032 return Register();
13033 if ((Subtarget->hasBWI())) {
13034 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13035 }
13036 return Register();
13037}
13038
13039Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13040 if (RetVT.SimpleTy != MVT::v8i16)
13041 return Register();
13042 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13043 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13044 }
13045 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13046 return fastEmitInst_rr(MachineInstOpcode: X86::PADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13047 }
13048 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13049 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13050 }
13051 return Register();
13052}
13053
13054Register fastEmit_ISD_SADDSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13055 if (RetVT.SimpleTy != MVT::v16i16)
13056 return Register();
13057 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13058 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13059 }
13060 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13061 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13062 }
13063 return Register();
13064}
13065
13066Register fastEmit_ISD_SADDSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13067 if (RetVT.SimpleTy != MVT::v32i16)
13068 return Register();
13069 if ((Subtarget->hasBWI())) {
13070 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13071 }
13072 return Register();
13073}
13074
13075Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13076 switch (VT.SimpleTy) {
13077 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13078 case MVT::v32i8: return fastEmit_ISD_SADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
13079 case MVT::v64i8: return fastEmit_ISD_SADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
13080 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13081 case MVT::v16i16: return fastEmit_ISD_SADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
13082 case MVT::v32i16: return fastEmit_ISD_SADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
13083 default: return Register();
13084 }
13085}
13086
13087// FastEmit functions for ISD::SHL.
13088
13089Register fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13090 if (RetVT.SimpleTy != MVT::i8)
13091 return Register();
13092 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
13093 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13094 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL_ND, RC: &X86::GR8RegClass, Op0);
13095 }
13096 if ((!Subtarget->hasNDD())) {
13097 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13098 return fastEmitInst_r(MachineInstOpcode: X86::SHL8rCL, RC: &X86::GR8RegClass, Op0);
13099 }
13100 return Register();
13101}
13102
13103Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13104 switch (VT.SimpleTy) {
13105 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op1);
13106 default: return Register();
13107 }
13108}
13109
13110// FastEmit functions for ISD::SMAX.
13111
13112Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13113 if (RetVT.SimpleTy != MVT::v16i8)
13114 return Register();
13115 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13116 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13117 }
13118 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13119 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13120 }
13121 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13122 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13123 }
13124 return Register();
13125}
13126
13127Register fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13128 if (RetVT.SimpleTy != MVT::v32i8)
13129 return Register();
13130 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13131 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13132 }
13133 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13134 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13135 }
13136 return Register();
13137}
13138
13139Register fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13140 if (RetVT.SimpleTy != MVT::v64i8)
13141 return Register();
13142 if ((Subtarget->hasBWI())) {
13143 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13144 }
13145 return Register();
13146}
13147
13148Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13149 if (RetVT.SimpleTy != MVT::v8i16)
13150 return Register();
13151 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13152 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13153 }
13154 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13155 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13156 }
13157 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13158 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13159 }
13160 return Register();
13161}
13162
13163Register fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13164 if (RetVT.SimpleTy != MVT::v16i16)
13165 return Register();
13166 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13167 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13168 }
13169 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13170 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13171 }
13172 return Register();
13173}
13174
13175Register fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13176 if (RetVT.SimpleTy != MVT::v32i16)
13177 return Register();
13178 if ((Subtarget->hasBWI())) {
13179 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13180 }
13181 return Register();
13182}
13183
13184Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13185 if (RetVT.SimpleTy != MVT::v4i32)
13186 return Register();
13187 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13188 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13189 }
13190 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13191 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
13192 }
13193 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13194 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDrr, RC: &X86::VR128RegClass, Op0, Op1);
13195 }
13196 return Register();
13197}
13198
13199Register fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
13200 if (RetVT.SimpleTy != MVT::v8i32)
13201 return Register();
13202 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13203 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13204 }
13205 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13206 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13207 }
13208 return Register();
13209}
13210
13211Register fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
13212 if (RetVT.SimpleTy != MVT::v16i32)
13213 return Register();
13214 if ((Subtarget->hasAVX512())) {
13215 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13216 }
13217 return Register();
13218}
13219
13220Register fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13221 if (RetVT.SimpleTy != MVT::v2i64)
13222 return Register();
13223 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13224 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13225 }
13226 return Register();
13227}
13228
13229Register fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
13230 if (RetVT.SimpleTy != MVT::v4i64)
13231 return Register();
13232 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13233 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13234 }
13235 return Register();
13236}
13237
13238Register fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
13239 if (RetVT.SimpleTy != MVT::v8i64)
13240 return Register();
13241 if ((Subtarget->hasAVX512())) {
13242 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13243 }
13244 return Register();
13245}
13246
13247Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13248 switch (VT.SimpleTy) {
13249 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13250 case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
13251 case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
13252 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13253 case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
13254 case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
13255 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13256 case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
13257 case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
13258 case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
13259 case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
13260 case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
13261 default: return Register();
13262 }
13263}
13264
13265// FastEmit functions for ISD::SMIN.
13266
13267Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13268 if (RetVT.SimpleTy != MVT::v16i8)
13269 return Register();
13270 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13271 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13272 }
13273 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13274 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13275 }
13276 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13277 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13278 }
13279 return Register();
13280}
13281
13282Register fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13283 if (RetVT.SimpleTy != MVT::v32i8)
13284 return Register();
13285 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13286 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13287 }
13288 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13289 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13290 }
13291 return Register();
13292}
13293
13294Register fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13295 if (RetVT.SimpleTy != MVT::v64i8)
13296 return Register();
13297 if ((Subtarget->hasBWI())) {
13298 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13299 }
13300 return Register();
13301}
13302
13303Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13304 if (RetVT.SimpleTy != MVT::v8i16)
13305 return Register();
13306 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13307 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13308 }
13309 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13310 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13311 }
13312 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13313 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13314 }
13315 return Register();
13316}
13317
13318Register fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13319 if (RetVT.SimpleTy != MVT::v16i16)
13320 return Register();
13321 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13322 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13323 }
13324 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13325 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13326 }
13327 return Register();
13328}
13329
13330Register fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13331 if (RetVT.SimpleTy != MVT::v32i16)
13332 return Register();
13333 if ((Subtarget->hasBWI())) {
13334 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13335 }
13336 return Register();
13337}
13338
13339Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13340 if (RetVT.SimpleTy != MVT::v4i32)
13341 return Register();
13342 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13343 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13344 }
13345 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13346 return fastEmitInst_rr(MachineInstOpcode: X86::PMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
13347 }
13348 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13349 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDrr, RC: &X86::VR128RegClass, Op0, Op1);
13350 }
13351 return Register();
13352}
13353
13354Register fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
13355 if (RetVT.SimpleTy != MVT::v8i32)
13356 return Register();
13357 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13358 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13359 }
13360 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13361 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13362 }
13363 return Register();
13364}
13365
13366Register fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
13367 if (RetVT.SimpleTy != MVT::v16i32)
13368 return Register();
13369 if ((Subtarget->hasAVX512())) {
13370 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13371 }
13372 return Register();
13373}
13374
13375Register fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13376 if (RetVT.SimpleTy != MVT::v2i64)
13377 return Register();
13378 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13379 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13380 }
13381 return Register();
13382}
13383
13384Register fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
13385 if (RetVT.SimpleTy != MVT::v4i64)
13386 return Register();
13387 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13388 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13389 }
13390 return Register();
13391}
13392
13393Register fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
13394 if (RetVT.SimpleTy != MVT::v8i64)
13395 return Register();
13396 if ((Subtarget->hasAVX512())) {
13397 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINSQZrr, RC: &X86::VR512RegClass, Op0, Op1);
13398 }
13399 return Register();
13400}
13401
13402Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13403 switch (VT.SimpleTy) {
13404 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13405 case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
13406 case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
13407 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13408 case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
13409 case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
13410 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13411 case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
13412 case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
13413 case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
13414 case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
13415 case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
13416 default: return Register();
13417 }
13418}
13419
13420// FastEmit functions for ISD::SRA.
13421
13422Register fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13423 if (RetVT.SimpleTy != MVT::i8)
13424 return Register();
13425 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
13426 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13427 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL_ND, RC: &X86::GR8RegClass, Op0);
13428 }
13429 if ((!Subtarget->hasNDD())) {
13430 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13431 return fastEmitInst_r(MachineInstOpcode: X86::SAR8rCL, RC: &X86::GR8RegClass, Op0);
13432 }
13433 return Register();
13434}
13435
13436Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13437 switch (VT.SimpleTy) {
13438 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op1);
13439 default: return Register();
13440 }
13441}
13442
13443// FastEmit functions for ISD::SRL.
13444
13445Register fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
13446 if (RetVT.SimpleTy != MVT::i8)
13447 return Register();
13448 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
13449 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13450 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL_ND, RC: &X86::GR8RegClass, Op0);
13451 }
13452 if ((!Subtarget->hasNDD())) {
13453 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::CL).addReg(RegNo: Op1);
13454 return fastEmitInst_r(MachineInstOpcode: X86::SHR8rCL, RC: &X86::GR8RegClass, Op0);
13455 }
13456 return Register();
13457}
13458
13459Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13460 switch (VT.SimpleTy) {
13461 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op1);
13462 default: return Register();
13463 }
13464}
13465
13466// FastEmit functions for ISD::SSUBSAT.
13467
13468Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13469 if (RetVT.SimpleTy != MVT::v16i8)
13470 return Register();
13471 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13472 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13473 }
13474 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13475 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13476 }
13477 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13478 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBrr, RC: &X86::VR128RegClass, Op0, Op1);
13479 }
13480 return Register();
13481}
13482
13483Register fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
13484 if (RetVT.SimpleTy != MVT::v32i8)
13485 return Register();
13486 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13487 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13488 }
13489 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13490 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
13491 }
13492 return Register();
13493}
13494
13495Register fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
13496 if (RetVT.SimpleTy != MVT::v64i8)
13497 return Register();
13498 if ((Subtarget->hasBWI())) {
13499 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
13500 }
13501 return Register();
13502}
13503
13504Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13505 if (RetVT.SimpleTy != MVT::v8i16)
13506 return Register();
13507 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13508 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13509 }
13510 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13511 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13512 }
13513 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13514 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
13515 }
13516 return Register();
13517}
13518
13519Register fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
13520 if (RetVT.SimpleTy != MVT::v16i16)
13521 return Register();
13522 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13523 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13524 }
13525 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13526 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
13527 }
13528 return Register();
13529}
13530
13531Register fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
13532 if (RetVT.SimpleTy != MVT::v32i16)
13533 return Register();
13534 if ((Subtarget->hasBWI())) {
13535 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
13536 }
13537 return Register();
13538}
13539
13540Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13541 switch (VT.SimpleTy) {
13542 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13543 case MVT::v32i8: return fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
13544 case MVT::v64i8: return fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
13545 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13546 case MVT::v16i16: return fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
13547 case MVT::v32i16: return fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
13548 default: return Register();
13549 }
13550}
13551
13552// FastEmit functions for ISD::STRICT_FADD.
13553
13554Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13555 if (RetVT.SimpleTy != MVT::f16)
13556 return Register();
13557 if ((Subtarget->hasFP16())) {
13558 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13559 }
13560 return Register();
13561}
13562
13563Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13564 if (RetVT.SimpleTy != MVT::f32)
13565 return Register();
13566 if ((Subtarget->hasAVX512())) {
13567 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13568 }
13569 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13570 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13571 }
13572 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13573 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13574 }
13575 if ((!Subtarget->hasSSE1())) {
13576 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13577 }
13578 return Register();
13579}
13580
13581Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13582 if (RetVT.SimpleTy != MVT::f64)
13583 return Register();
13584 if ((Subtarget->hasAVX512())) {
13585 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13586 }
13587 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13588 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13589 }
13590 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13591 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13592 }
13593 if ((!Subtarget->hasSSE2())) {
13594 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13595 }
13596 return Register();
13597}
13598
13599Register fastEmit_ISD_STRICT_FADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13600 if (RetVT.SimpleTy != MVT::f80)
13601 return Register();
13602 if ((Subtarget->hasX87())) {
13603 return fastEmitInst_rr(MachineInstOpcode: X86::ADD_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13604 }
13605 return Register();
13606}
13607
13608Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13609 if (RetVT.SimpleTy != MVT::v8f16)
13610 return Register();
13611 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13612 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13613 }
13614 return Register();
13615}
13616
13617Register fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13618 if (RetVT.SimpleTy != MVT::v16f16)
13619 return Register();
13620 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13621 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13622 }
13623 return Register();
13624}
13625
13626Register fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13627 if (RetVT.SimpleTy != MVT::v32f16)
13628 return Register();
13629 if ((Subtarget->hasFP16())) {
13630 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13631 }
13632 return Register();
13633}
13634
13635Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13636 if (RetVT.SimpleTy != MVT::v4f32)
13637 return Register();
13638 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13639 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13640 }
13641 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13642 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13643 }
13644 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13645 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13646 }
13647 return Register();
13648}
13649
13650Register fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13651 if (RetVT.SimpleTy != MVT::v8f32)
13652 return Register();
13653 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13654 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13655 }
13656 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13657 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13658 }
13659 return Register();
13660}
13661
13662Register fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13663 if (RetVT.SimpleTy != MVT::v16f32)
13664 return Register();
13665 if ((Subtarget->hasAVX512())) {
13666 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13667 }
13668 return Register();
13669}
13670
13671Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13672 if (RetVT.SimpleTy != MVT::v2f64)
13673 return Register();
13674 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13675 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13676 }
13677 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13678 return fastEmitInst_rr(MachineInstOpcode: X86::ADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13679 }
13680 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13681 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13682 }
13683 return Register();
13684}
13685
13686Register fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13687 if (RetVT.SimpleTy != MVT::v4f64)
13688 return Register();
13689 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13690 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13691 }
13692 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13693 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13694 }
13695 return Register();
13696}
13697
13698Register fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13699 if (RetVT.SimpleTy != MVT::v8f64)
13700 return Register();
13701 if ((Subtarget->hasAVX512())) {
13702 return fastEmitInst_rr(MachineInstOpcode: X86::VADDPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13703 }
13704 return Register();
13705}
13706
13707Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13708 switch (VT.SimpleTy) {
13709 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
13710 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
13711 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
13712 case MVT::f80: return fastEmit_ISD_STRICT_FADD_MVT_f80_rr(RetVT, Op0, Op1);
13713 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
13714 case MVT::v16f16: return fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
13715 case MVT::v32f16: return fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
13716 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
13717 case MVT::v8f32: return fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
13718 case MVT::v16f32: return fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
13719 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
13720 case MVT::v4f64: return fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
13721 case MVT::v8f64: return fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
13722 default: return Register();
13723 }
13724}
13725
13726// FastEmit functions for ISD::STRICT_FDIV.
13727
13728Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13729 if (RetVT.SimpleTy != MVT::f16)
13730 return Register();
13731 if ((Subtarget->hasFP16())) {
13732 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13733 }
13734 return Register();
13735}
13736
13737Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13738 if (RetVT.SimpleTy != MVT::f32)
13739 return Register();
13740 if ((Subtarget->hasAVX512())) {
13741 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13742 }
13743 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13744 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13745 }
13746 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13747 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13748 }
13749 if ((!Subtarget->hasSSE1())) {
13750 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13751 }
13752 return Register();
13753}
13754
13755Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13756 if (RetVT.SimpleTy != MVT::f64)
13757 return Register();
13758 if ((Subtarget->hasAVX512())) {
13759 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13760 }
13761 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13762 return fastEmitInst_rr(MachineInstOpcode: X86::DIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13763 }
13764 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13765 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13766 }
13767 if ((!Subtarget->hasSSE2())) {
13768 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13769 }
13770 return Register();
13771}
13772
13773Register fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13774 if (RetVT.SimpleTy != MVT::f80)
13775 return Register();
13776 if ((Subtarget->hasX87())) {
13777 return fastEmitInst_rr(MachineInstOpcode: X86::DIV_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13778 }
13779 return Register();
13780}
13781
13782Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13783 if (RetVT.SimpleTy != MVT::v8f16)
13784 return Register();
13785 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13786 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13787 }
13788 return Register();
13789}
13790
13791Register fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13792 if (RetVT.SimpleTy != MVT::v16f16)
13793 return Register();
13794 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13795 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13796 }
13797 return Register();
13798}
13799
13800Register fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13801 if (RetVT.SimpleTy != MVT::v32f16)
13802 return Register();
13803 if ((Subtarget->hasFP16())) {
13804 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13805 }
13806 return Register();
13807}
13808
13809Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13810 if (RetVT.SimpleTy != MVT::v4f32)
13811 return Register();
13812 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13813 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13814 }
13815 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13816 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13817 }
13818 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13819 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13820 }
13821 return Register();
13822}
13823
13824Register fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13825 if (RetVT.SimpleTy != MVT::v8f32)
13826 return Register();
13827 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13828 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13829 }
13830 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13831 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
13832 }
13833 return Register();
13834}
13835
13836Register fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
13837 if (RetVT.SimpleTy != MVT::v16f32)
13838 return Register();
13839 if ((Subtarget->hasAVX512())) {
13840 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
13841 }
13842 return Register();
13843}
13844
13845Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13846 if (RetVT.SimpleTy != MVT::v2f64)
13847 return Register();
13848 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13849 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13850 }
13851 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13852 return fastEmitInst_rr(MachineInstOpcode: X86::DIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13853 }
13854 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13855 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDrr, RC: &X86::VR128RegClass, Op0, Op1);
13856 }
13857 return Register();
13858}
13859
13860Register fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
13861 if (RetVT.SimpleTy != MVT::v4f64)
13862 return Register();
13863 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13864 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13865 }
13866 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13867 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
13868 }
13869 return Register();
13870}
13871
13872Register fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
13873 if (RetVT.SimpleTy != MVT::v8f64)
13874 return Register();
13875 if ((Subtarget->hasAVX512())) {
13876 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
13877 }
13878 return Register();
13879}
13880
13881Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13882 switch (VT.SimpleTy) {
13883 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
13884 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
13885 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
13886 case MVT::f80: return fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
13887 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
13888 case MVT::v16f16: return fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
13889 case MVT::v32f16: return fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
13890 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
13891 case MVT::v8f32: return fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
13892 case MVT::v16f32: return fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
13893 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
13894 case MVT::v4f64: return fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
13895 case MVT::v8f64: return fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
13896 default: return Register();
13897 }
13898}
13899
13900// FastEmit functions for ISD::STRICT_FMUL.
13901
13902Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13903 if (RetVT.SimpleTy != MVT::f16)
13904 return Register();
13905 if ((Subtarget->hasFP16())) {
13906 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
13907 }
13908 return Register();
13909}
13910
13911Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13912 if (RetVT.SimpleTy != MVT::f32)
13913 return Register();
13914 if ((Subtarget->hasAVX512())) {
13915 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
13916 }
13917 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13918 return fastEmitInst_rr(MachineInstOpcode: X86::MULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13919 }
13920 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13921 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSrr, RC: &X86::FR32RegClass, Op0, Op1);
13922 }
13923 if ((!Subtarget->hasSSE1())) {
13924 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
13925 }
13926 return Register();
13927}
13928
13929Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13930 if (RetVT.SimpleTy != MVT::f64)
13931 return Register();
13932 if ((Subtarget->hasAVX512())) {
13933 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
13934 }
13935 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13936 return fastEmitInst_rr(MachineInstOpcode: X86::MULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13937 }
13938 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13939 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDrr, RC: &X86::FR64RegClass, Op0, Op1);
13940 }
13941 if ((!Subtarget->hasSSE2())) {
13942 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
13943 }
13944 return Register();
13945}
13946
13947Register fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
13948 if (RetVT.SimpleTy != MVT::f80)
13949 return Register();
13950 if ((Subtarget->hasX87())) {
13951 return fastEmitInst_rr(MachineInstOpcode: X86::MUL_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
13952 }
13953 return Register();
13954}
13955
13956Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13957 if (RetVT.SimpleTy != MVT::v8f16)
13958 return Register();
13959 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13960 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13961 }
13962 return Register();
13963}
13964
13965Register fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
13966 if (RetVT.SimpleTy != MVT::v16f16)
13967 return Register();
13968 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13969 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
13970 }
13971 return Register();
13972}
13973
13974Register fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
13975 if (RetVT.SimpleTy != MVT::v32f16)
13976 return Register();
13977 if ((Subtarget->hasFP16())) {
13978 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
13979 }
13980 return Register();
13981}
13982
13983Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13984 if (RetVT.SimpleTy != MVT::v4f32)
13985 return Register();
13986 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13987 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
13988 }
13989 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13990 return fastEmitInst_rr(MachineInstOpcode: X86::MULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13991 }
13992 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13993 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSrr, RC: &X86::VR128RegClass, Op0, Op1);
13994 }
13995 return Register();
13996}
13997
13998Register fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
13999 if (RetVT.SimpleTy != MVT::v8f32)
14000 return Register();
14001 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14002 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14003 }
14004 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14005 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14006 }
14007 return Register();
14008}
14009
14010Register fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
14011 if (RetVT.SimpleTy != MVT::v16f32)
14012 return Register();
14013 if ((Subtarget->hasAVX512())) {
14014 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14015 }
14016 return Register();
14017}
14018
14019Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14020 if (RetVT.SimpleTy != MVT::v2f64)
14021 return Register();
14022 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14023 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14024 }
14025 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14026 return fastEmitInst_rr(MachineInstOpcode: X86::MULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14027 }
14028 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14029 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14030 }
14031 return Register();
14032}
14033
14034Register fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14035 if (RetVT.SimpleTy != MVT::v4f64)
14036 return Register();
14037 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14038 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14039 }
14040 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14041 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14042 }
14043 return Register();
14044}
14045
14046Register fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
14047 if (RetVT.SimpleTy != MVT::v8f64)
14048 return Register();
14049 if ((Subtarget->hasAVX512())) {
14050 return fastEmitInst_rr(MachineInstOpcode: X86::VMULPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14051 }
14052 return Register();
14053}
14054
14055Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14056 switch (VT.SimpleTy) {
14057 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
14058 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
14059 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
14060 case MVT::f80: return fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
14061 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
14062 case MVT::v16f16: return fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
14063 case MVT::v32f16: return fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
14064 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
14065 case MVT::v8f32: return fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
14066 case MVT::v16f32: return fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
14067 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
14068 case MVT::v4f64: return fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
14069 case MVT::v8f64: return fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
14070 default: return Register();
14071 }
14072}
14073
14074// FastEmit functions for ISD::STRICT_FSUB.
14075
14076Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
14077 if (RetVT.SimpleTy != MVT::f16)
14078 return Register();
14079 if ((Subtarget->hasFP16())) {
14080 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
14081 }
14082 return Register();
14083}
14084
14085Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
14086 if (RetVT.SimpleTy != MVT::f32)
14087 return Register();
14088 if ((Subtarget->hasAVX512())) {
14089 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
14090 }
14091 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14092 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14093 }
14094 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14095 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14096 }
14097 if ((!Subtarget->hasSSE1())) {
14098 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp32, RC: &X86::RFP32RegClass, Op0, Op1);
14099 }
14100 return Register();
14101}
14102
14103Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
14104 if (RetVT.SimpleTy != MVT::f64)
14105 return Register();
14106 if ((Subtarget->hasAVX512())) {
14107 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
14108 }
14109 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14110 return fastEmitInst_rr(MachineInstOpcode: X86::SUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14111 }
14112 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14113 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14114 }
14115 if ((!Subtarget->hasSSE2())) {
14116 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp64, RC: &X86::RFP64RegClass, Op0, Op1);
14117 }
14118 return Register();
14119}
14120
14121Register fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
14122 if (RetVT.SimpleTy != MVT::f80)
14123 return Register();
14124 if ((Subtarget->hasX87())) {
14125 return fastEmitInst_rr(MachineInstOpcode: X86::SUB_Fp80, RC: &X86::RFP80RegClass, Op0, Op1);
14126 }
14127 return Register();
14128}
14129
14130Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
14131 if (RetVT.SimpleTy != MVT::v8f16)
14132 return Register();
14133 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14134 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14135 }
14136 return Register();
14137}
14138
14139Register fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
14140 if (RetVT.SimpleTy != MVT::v16f16)
14141 return Register();
14142 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14143 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14144 }
14145 return Register();
14146}
14147
14148Register fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
14149 if (RetVT.SimpleTy != MVT::v32f16)
14150 return Register();
14151 if ((Subtarget->hasFP16())) {
14152 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
14153 }
14154 return Register();
14155}
14156
14157Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
14158 if (RetVT.SimpleTy != MVT::v4f32)
14159 return Register();
14160 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14161 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14162 }
14163 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14164 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14165 }
14166 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14167 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14168 }
14169 return Register();
14170}
14171
14172Register fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
14173 if (RetVT.SimpleTy != MVT::v8f32)
14174 return Register();
14175 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14176 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14177 }
14178 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14179 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14180 }
14181 return Register();
14182}
14183
14184Register fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
14185 if (RetVT.SimpleTy != MVT::v16f32)
14186 return Register();
14187 if ((Subtarget->hasAVX512())) {
14188 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14189 }
14190 return Register();
14191}
14192
14193Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14194 if (RetVT.SimpleTy != MVT::v2f64)
14195 return Register();
14196 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14197 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14198 }
14199 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14200 return fastEmitInst_rr(MachineInstOpcode: X86::SUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14201 }
14202 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14203 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14204 }
14205 return Register();
14206}
14207
14208Register fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14209 if (RetVT.SimpleTy != MVT::v4f64)
14210 return Register();
14211 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14212 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14213 }
14214 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14215 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14216 }
14217 return Register();
14218}
14219
14220Register fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
14221 if (RetVT.SimpleTy != MVT::v8f64)
14222 return Register();
14223 if ((Subtarget->hasAVX512())) {
14224 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14225 }
14226 return Register();
14227}
14228
14229Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14230 switch (VT.SimpleTy) {
14231 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
14232 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
14233 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
14234 case MVT::f80: return fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
14235 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
14236 case MVT::v16f16: return fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
14237 case MVT::v32f16: return fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
14238 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
14239 case MVT::v8f32: return fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
14240 case MVT::v16f32: return fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
14241 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
14242 case MVT::v4f64: return fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
14243 case MVT::v8f64: return fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
14244 default: return Register();
14245 }
14246}
14247
14248// FastEmit functions for ISD::STRICT_PSEUDO_FMAX.
14249
14250Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
14251 if (RetVT.SimpleTy != MVT::f16)
14252 return Register();
14253 if ((Subtarget->hasFP16())) {
14254 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
14255 }
14256 return Register();
14257}
14258
14259Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
14260 if (RetVT.SimpleTy != MVT::f32)
14261 return Register();
14262 if ((Subtarget->hasAVX512())) {
14263 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
14264 }
14265 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14266 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14267 }
14268 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14269 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14270 }
14271 return Register();
14272}
14273
14274Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
14275 if (RetVT.SimpleTy != MVT::f64)
14276 return Register();
14277 if ((Subtarget->hasAVX512())) {
14278 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
14279 }
14280 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14281 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14282 }
14283 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14284 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14285 }
14286 return Register();
14287}
14288
14289Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
14290 if (RetVT.SimpleTy != MVT::v8f16)
14291 return Register();
14292 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14293 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14294 }
14295 return Register();
14296}
14297
14298Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
14299 if (RetVT.SimpleTy != MVT::v16f16)
14300 return Register();
14301 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14302 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14303 }
14304 return Register();
14305}
14306
14307Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
14308 if (RetVT.SimpleTy != MVT::v32f16)
14309 return Register();
14310 if ((Subtarget->hasFP16())) {
14311 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
14312 }
14313 return Register();
14314}
14315
14316Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
14317 if (RetVT.SimpleTy != MVT::v4f32)
14318 return Register();
14319 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14320 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14321 }
14322 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14323 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14324 }
14325 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14326 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14327 }
14328 return Register();
14329}
14330
14331Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
14332 if (RetVT.SimpleTy != MVT::v8f32)
14333 return Register();
14334 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14335 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14336 }
14337 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14338 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14339 }
14340 return Register();
14341}
14342
14343Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
14344 if (RetVT.SimpleTy != MVT::v16f32)
14345 return Register();
14346 if ((Subtarget->hasAVX512())) {
14347 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14348 }
14349 return Register();
14350}
14351
14352Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14353 if (RetVT.SimpleTy != MVT::v2f64)
14354 return Register();
14355 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14356 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14357 }
14358 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14359 return fastEmitInst_rr(MachineInstOpcode: X86::MAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14360 }
14361 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14362 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14363 }
14364 return Register();
14365}
14366
14367Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14368 if (RetVT.SimpleTy != MVT::v4f64)
14369 return Register();
14370 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14371 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14372 }
14373 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14374 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14375 }
14376 return Register();
14377}
14378
14379Register fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
14380 if (RetVT.SimpleTy != MVT::v8f64)
14381 return Register();
14382 if ((Subtarget->hasAVX512())) {
14383 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14384 }
14385 return Register();
14386}
14387
14388Register fastEmit_ISD_STRICT_PSEUDO_FMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14389 switch (VT.SimpleTy) {
14390 case MVT::f16: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
14391 case MVT::f32: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
14392 case MVT::f64: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
14393 case MVT::v8f16: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
14394 case MVT::v16f16: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
14395 case MVT::v32f16: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
14396 case MVT::v4f32: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
14397 case MVT::v8f32: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
14398 case MVT::v16f32: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
14399 case MVT::v2f64: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
14400 case MVT::v4f64: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
14401 case MVT::v8f64: return fastEmit_ISD_STRICT_PSEUDO_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
14402 default: return Register();
14403 }
14404}
14405
14406// FastEmit functions for ISD::STRICT_PSEUDO_FMIN.
14407
14408Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
14409 if (RetVT.SimpleTy != MVT::f16)
14410 return Register();
14411 if ((Subtarget->hasFP16())) {
14412 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
14413 }
14414 return Register();
14415}
14416
14417Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
14418 if (RetVT.SimpleTy != MVT::f32)
14419 return Register();
14420 if ((Subtarget->hasAVX512())) {
14421 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
14422 }
14423 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14424 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14425 }
14426 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14427 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr, RC: &X86::FR32RegClass, Op0, Op1);
14428 }
14429 return Register();
14430}
14431
14432Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
14433 if (RetVT.SimpleTy != MVT::f64)
14434 return Register();
14435 if ((Subtarget->hasAVX512())) {
14436 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
14437 }
14438 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14439 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14440 }
14441 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14442 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr, RC: &X86::FR64RegClass, Op0, Op1);
14443 }
14444 return Register();
14445}
14446
14447Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
14448 if (RetVT.SimpleTy != MVT::v8f16)
14449 return Register();
14450 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14451 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14452 }
14453 return Register();
14454}
14455
14456Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
14457 if (RetVT.SimpleTy != MVT::v16f16)
14458 return Register();
14459 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14460 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14461 }
14462 return Register();
14463}
14464
14465Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
14466 if (RetVT.SimpleTy != MVT::v32f16)
14467 return Register();
14468 if ((Subtarget->hasFP16())) {
14469 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
14470 }
14471 return Register();
14472}
14473
14474Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
14475 if (RetVT.SimpleTy != MVT::v4f32)
14476 return Register();
14477 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14478 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14479 }
14480 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14481 return fastEmitInst_rr(MachineInstOpcode: X86::MINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14482 }
14483 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14484 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSrr, RC: &X86::VR128RegClass, Op0, Op1);
14485 }
14486 return Register();
14487}
14488
14489Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
14490 if (RetVT.SimpleTy != MVT::v8f32)
14491 return Register();
14492 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14493 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14494 }
14495 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14496 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
14497 }
14498 return Register();
14499}
14500
14501Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
14502 if (RetVT.SimpleTy != MVT::v16f32)
14503 return Register();
14504 if ((Subtarget->hasAVX512())) {
14505 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
14506 }
14507 return Register();
14508}
14509
14510Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
14511 if (RetVT.SimpleTy != MVT::v2f64)
14512 return Register();
14513 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14514 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14515 }
14516 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14517 return fastEmitInst_rr(MachineInstOpcode: X86::MINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14518 }
14519 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14520 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDrr, RC: &X86::VR128RegClass, Op0, Op1);
14521 }
14522 return Register();
14523}
14524
14525Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
14526 if (RetVT.SimpleTy != MVT::v4f64)
14527 return Register();
14528 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14529 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14530 }
14531 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14532 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14533 }
14534 return Register();
14535}
14536
14537Register fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
14538 if (RetVT.SimpleTy != MVT::v8f64)
14539 return Register();
14540 if ((Subtarget->hasAVX512())) {
14541 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14542 }
14543 return Register();
14544}
14545
14546Register fastEmit_ISD_STRICT_PSEUDO_FMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14547 switch (VT.SimpleTy) {
14548 case MVT::f16: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
14549 case MVT::f32: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
14550 case MVT::f64: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
14551 case MVT::v8f16: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
14552 case MVT::v16f16: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
14553 case MVT::v32f16: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
14554 case MVT::v4f32: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
14555 case MVT::v8f32: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
14556 case MVT::v16f32: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
14557 case MVT::v2f64: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
14558 case MVT::v4f64: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
14559 case MVT::v8f64: return fastEmit_ISD_STRICT_PSEUDO_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
14560 default: return Register();
14561 }
14562}
14563
14564// FastEmit functions for ISD::SUB.
14565
14566Register fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
14567 if (RetVT.SimpleTy != MVT::i8)
14568 return Register();
14569 if ((Subtarget->hasNDD())) {
14570 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
14571 }
14572 if ((!Subtarget->hasNDD())) {
14573 return fastEmitInst_rr(MachineInstOpcode: X86::SUB8rr, RC: &X86::GR8RegClass, Op0, Op1);
14574 }
14575 return Register();
14576}
14577
14578Register fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
14579 if (RetVT.SimpleTy != MVT::i16)
14580 return Register();
14581 if ((Subtarget->hasNDD())) {
14582 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
14583 }
14584 if ((!Subtarget->hasNDD())) {
14585 return fastEmitInst_rr(MachineInstOpcode: X86::SUB16rr, RC: &X86::GR16RegClass, Op0, Op1);
14586 }
14587 return Register();
14588}
14589
14590Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14591 if (RetVT.SimpleTy != MVT::i32)
14592 return Register();
14593 if ((Subtarget->hasNDD())) {
14594 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
14595 }
14596 if ((!Subtarget->hasNDD())) {
14597 return fastEmitInst_rr(MachineInstOpcode: X86::SUB32rr, RC: &X86::GR32RegClass, Op0, Op1);
14598 }
14599 return Register();
14600}
14601
14602Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14603 if (RetVT.SimpleTy != MVT::i64)
14604 return Register();
14605 if ((Subtarget->hasNDD())) {
14606 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
14607 }
14608 if ((!Subtarget->hasNDD())) {
14609 return fastEmitInst_rr(MachineInstOpcode: X86::SUB64rr, RC: &X86::GR64RegClass, Op0, Op1);
14610 }
14611 return Register();
14612}
14613
14614Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14615 if (RetVT.SimpleTy != MVT::v16i8)
14616 return Register();
14617 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14618 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14619 }
14620 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14621 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
14622 }
14623 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14624 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBrr, RC: &X86::VR128RegClass, Op0, Op1);
14625 }
14626 return Register();
14627}
14628
14629Register fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14630 if (RetVT.SimpleTy != MVT::v32i8)
14631 return Register();
14632 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14633 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14634 }
14635 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14636 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14637 }
14638 return Register();
14639}
14640
14641Register fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14642 if (RetVT.SimpleTy != MVT::v64i8)
14643 return Register();
14644 if ((Subtarget->hasBWI())) {
14645 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14646 }
14647 return Register();
14648}
14649
14650Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14651 if (RetVT.SimpleTy != MVT::v8i16)
14652 return Register();
14653 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14654 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14655 }
14656 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14657 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
14658 }
14659 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14660 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
14661 }
14662 return Register();
14663}
14664
14665Register fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14666 if (RetVT.SimpleTy != MVT::v16i16)
14667 return Register();
14668 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14669 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14670 }
14671 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14672 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14673 }
14674 return Register();
14675}
14676
14677Register fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14678 if (RetVT.SimpleTy != MVT::v32i16)
14679 return Register();
14680 if ((Subtarget->hasBWI())) {
14681 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14682 }
14683 return Register();
14684}
14685
14686Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14687 if (RetVT.SimpleTy != MVT::v4i32)
14688 return Register();
14689 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14690 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14691 }
14692 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14693 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
14694 }
14695 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14696 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
14697 }
14698 return Register();
14699}
14700
14701Register fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14702 if (RetVT.SimpleTy != MVT::v8i32)
14703 return Register();
14704 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14705 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14706 }
14707 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14708 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14709 }
14710 return Register();
14711}
14712
14713Register fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14714 if (RetVT.SimpleTy != MVT::v16i32)
14715 return Register();
14716 if ((Subtarget->hasAVX512())) {
14717 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14718 }
14719 return Register();
14720}
14721
14722Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14723 if (RetVT.SimpleTy != MVT::v2i64)
14724 return Register();
14725 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14726 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14727 }
14728 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14729 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
14730 }
14731 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14732 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQrr, RC: &X86::VR128RegClass, Op0, Op1);
14733 }
14734 return Register();
14735}
14736
14737Register fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14738 if (RetVT.SimpleTy != MVT::v4i64)
14739 return Register();
14740 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14741 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14742 }
14743 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14744 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQYrr, RC: &X86::VR256RegClass, Op0, Op1);
14745 }
14746 return Register();
14747}
14748
14749Register fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14750 if (RetVT.SimpleTy != MVT::v8i64)
14751 return Register();
14752 if ((Subtarget->hasAVX512())) {
14753 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14754 }
14755 return Register();
14756}
14757
14758Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14759 switch (VT.SimpleTy) {
14760 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op1);
14761 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op1);
14762 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
14763 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
14764 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
14765 case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op1);
14766 case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op1);
14767 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
14768 case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op1);
14769 case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op1);
14770 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
14771 case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op1);
14772 case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op1);
14773 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
14774 case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op1);
14775 case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op1);
14776 default: return Register();
14777 }
14778}
14779
14780// FastEmit functions for ISD::UADDSAT.
14781
14782Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14783 if (RetVT.SimpleTy != MVT::v16i8)
14784 return Register();
14785 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14786 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14787 }
14788 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14789 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14790 }
14791 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14792 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
14793 }
14794 return Register();
14795}
14796
14797Register fastEmit_ISD_UADDSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14798 if (RetVT.SimpleTy != MVT::v32i8)
14799 return Register();
14800 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14801 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14802 }
14803 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14804 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14805 }
14806 return Register();
14807}
14808
14809Register fastEmit_ISD_UADDSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14810 if (RetVT.SimpleTy != MVT::v64i8)
14811 return Register();
14812 if ((Subtarget->hasBWI())) {
14813 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14814 }
14815 return Register();
14816}
14817
14818Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14819 if (RetVT.SimpleTy != MVT::v8i16)
14820 return Register();
14821 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14822 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14823 }
14824 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14825 return fastEmitInst_rr(MachineInstOpcode: X86::PADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14826 }
14827 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14828 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
14829 }
14830 return Register();
14831}
14832
14833Register fastEmit_ISD_UADDSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14834 if (RetVT.SimpleTy != MVT::v16i16)
14835 return Register();
14836 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14837 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14838 }
14839 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14840 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14841 }
14842 return Register();
14843}
14844
14845Register fastEmit_ISD_UADDSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14846 if (RetVT.SimpleTy != MVT::v32i16)
14847 return Register();
14848 if ((Subtarget->hasBWI())) {
14849 return fastEmitInst_rr(MachineInstOpcode: X86::VPADDUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14850 }
14851 return Register();
14852}
14853
14854Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14855 switch (VT.SimpleTy) {
14856 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14857 case MVT::v32i8: return fastEmit_ISD_UADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
14858 case MVT::v64i8: return fastEmit_ISD_UADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
14859 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14860 case MVT::v16i16: return fastEmit_ISD_UADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
14861 case MVT::v32i16: return fastEmit_ISD_UADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
14862 default: return Register();
14863 }
14864}
14865
14866// FastEmit functions for ISD::UMAX.
14867
14868Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14869 if (RetVT.SimpleTy != MVT::v16i8)
14870 return Register();
14871 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14872 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14873 }
14874 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14875 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14876 }
14877 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14878 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBrr, RC: &X86::VR128RegClass, Op0, Op1);
14879 }
14880 return Register();
14881}
14882
14883Register fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
14884 if (RetVT.SimpleTy != MVT::v32i8)
14885 return Register();
14886 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14887 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14888 }
14889 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14890 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
14891 }
14892 return Register();
14893}
14894
14895Register fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
14896 if (RetVT.SimpleTy != MVT::v64i8)
14897 return Register();
14898 if ((Subtarget->hasBWI())) {
14899 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
14900 }
14901 return Register();
14902}
14903
14904Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14905 if (RetVT.SimpleTy != MVT::v8i16)
14906 return Register();
14907 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14908 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14909 }
14910 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14911 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14912 }
14913 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14914 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWrr, RC: &X86::VR128RegClass, Op0, Op1);
14915 }
14916 return Register();
14917}
14918
14919Register fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
14920 if (RetVT.SimpleTy != MVT::v16i16)
14921 return Register();
14922 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14923 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14924 }
14925 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14926 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
14927 }
14928 return Register();
14929}
14930
14931Register fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
14932 if (RetVT.SimpleTy != MVT::v32i16)
14933 return Register();
14934 if ((Subtarget->hasBWI())) {
14935 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
14936 }
14937 return Register();
14938}
14939
14940Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14941 if (RetVT.SimpleTy != MVT::v4i32)
14942 return Register();
14943 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14944 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14945 }
14946 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14947 return fastEmitInst_rr(MachineInstOpcode: X86::PMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14948 }
14949 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14950 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDrr, RC: &X86::VR128RegClass, Op0, Op1);
14951 }
14952 return Register();
14953}
14954
14955Register fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
14956 if (RetVT.SimpleTy != MVT::v8i32)
14957 return Register();
14958 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14959 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14960 }
14961 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14962 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
14963 }
14964 return Register();
14965}
14966
14967Register fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
14968 if (RetVT.SimpleTy != MVT::v16i32)
14969 return Register();
14970 if ((Subtarget->hasAVX512())) {
14971 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
14972 }
14973 return Register();
14974}
14975
14976Register fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14977 if (RetVT.SimpleTy != MVT::v2i64)
14978 return Register();
14979 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14980 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
14981 }
14982 return Register();
14983}
14984
14985Register fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
14986 if (RetVT.SimpleTy != MVT::v4i64)
14987 return Register();
14988 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14989 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
14990 }
14991 return Register();
14992}
14993
14994Register fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
14995 if (RetVT.SimpleTy != MVT::v8i64)
14996 return Register();
14997 if ((Subtarget->hasAVX512())) {
14998 return fastEmitInst_rr(MachineInstOpcode: X86::VPMAXUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
14999 }
15000 return Register();
15001}
15002
15003Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15004 switch (VT.SimpleTy) {
15005 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
15006 case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
15007 case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
15008 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
15009 case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
15010 case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
15011 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
15012 case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
15013 case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
15014 case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
15015 case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
15016 case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
15017 default: return Register();
15018 }
15019}
15020
15021// FastEmit functions for ISD::UMIN.
15022
15023Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
15024 if (RetVT.SimpleTy != MVT::v16i8)
15025 return Register();
15026 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15027 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15028 }
15029 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15030 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
15031 }
15032 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15033 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBrr, RC: &X86::VR128RegClass, Op0, Op1);
15034 }
15035 return Register();
15036}
15037
15038Register fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
15039 if (RetVT.SimpleTy != MVT::v32i8)
15040 return Register();
15041 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15042 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15043 }
15044 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15045 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBYrr, RC: &X86::VR256RegClass, Op0, Op1);
15046 }
15047 return Register();
15048}
15049
15050Register fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
15051 if (RetVT.SimpleTy != MVT::v64i8)
15052 return Register();
15053 if ((Subtarget->hasBWI())) {
15054 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUBZrr, RC: &X86::VR512RegClass, Op0, Op1);
15055 }
15056 return Register();
15057}
15058
15059Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
15060 if (RetVT.SimpleTy != MVT::v8i16)
15061 return Register();
15062 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15063 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15064 }
15065 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15066 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
15067 }
15068 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15069 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWrr, RC: &X86::VR128RegClass, Op0, Op1);
15070 }
15071 return Register();
15072}
15073
15074Register fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
15075 if (RetVT.SimpleTy != MVT::v16i16)
15076 return Register();
15077 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15078 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15079 }
15080 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15081 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15082 }
15083 return Register();
15084}
15085
15086Register fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
15087 if (RetVT.SimpleTy != MVT::v32i16)
15088 return Register();
15089 if ((Subtarget->hasBWI())) {
15090 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUWZrr, RC: &X86::VR512RegClass, Op0, Op1);
15091 }
15092 return Register();
15093}
15094
15095Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
15096 if (RetVT.SimpleTy != MVT::v4i32)
15097 return Register();
15098 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15099 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15100 }
15101 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15102 return fastEmitInst_rr(MachineInstOpcode: X86::PMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
15103 }
15104 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15105 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDrr, RC: &X86::VR128RegClass, Op0, Op1);
15106 }
15107 return Register();
15108}
15109
15110Register fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
15111 if (RetVT.SimpleTy != MVT::v8i32)
15112 return Register();
15113 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15114 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15115 }
15116 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15117 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15118 }
15119 return Register();
15120}
15121
15122Register fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
15123 if (RetVT.SimpleTy != MVT::v16i32)
15124 return Register();
15125 if ((Subtarget->hasAVX512())) {
15126 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15127 }
15128 return Register();
15129}
15130
15131Register fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
15132 if (RetVT.SimpleTy != MVT::v2i64)
15133 return Register();
15134 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15135 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15136 }
15137 return Register();
15138}
15139
15140Register fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
15141 if (RetVT.SimpleTy != MVT::v4i64)
15142 return Register();
15143 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15144 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15145 }
15146 return Register();
15147}
15148
15149Register fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
15150 if (RetVT.SimpleTy != MVT::v8i64)
15151 return Register();
15152 if ((Subtarget->hasAVX512())) {
15153 return fastEmitInst_rr(MachineInstOpcode: X86::VPMINUQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15154 }
15155 return Register();
15156}
15157
15158Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15159 switch (VT.SimpleTy) {
15160 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
15161 case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
15162 case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
15163 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
15164 case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
15165 case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
15166 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
15167 case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
15168 case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
15169 case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
15170 case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
15171 case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
15172 default: return Register();
15173 }
15174}
15175
15176// FastEmit functions for ISD::USUBSAT.
15177
15178Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
15179 if (RetVT.SimpleTy != MVT::v16i8)
15180 return Register();
15181 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15182 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15183 }
15184 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15185 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
15186 }
15187 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15188 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBrr, RC: &X86::VR128RegClass, Op0, Op1);
15189 }
15190 return Register();
15191}
15192
15193Register fastEmit_ISD_USUBSAT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
15194 if (RetVT.SimpleTy != MVT::v32i8)
15195 return Register();
15196 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15197 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15198 }
15199 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15200 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBYrr, RC: &X86::VR256RegClass, Op0, Op1);
15201 }
15202 return Register();
15203}
15204
15205Register fastEmit_ISD_USUBSAT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
15206 if (RetVT.SimpleTy != MVT::v64i8)
15207 return Register();
15208 if ((Subtarget->hasBWI())) {
15209 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSBZrr, RC: &X86::VR512RegClass, Op0, Op1);
15210 }
15211 return Register();
15212}
15213
15214Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
15215 if (RetVT.SimpleTy != MVT::v8i16)
15216 return Register();
15217 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15218 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15219 }
15220 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15221 return fastEmitInst_rr(MachineInstOpcode: X86::PSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
15222 }
15223 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15224 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWrr, RC: &X86::VR128RegClass, Op0, Op1);
15225 }
15226 return Register();
15227}
15228
15229Register fastEmit_ISD_USUBSAT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
15230 if (RetVT.SimpleTy != MVT::v16i16)
15231 return Register();
15232 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15233 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15234 }
15235 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15236 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
15237 }
15238 return Register();
15239}
15240
15241Register fastEmit_ISD_USUBSAT_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
15242 if (RetVT.SimpleTy != MVT::v32i16)
15243 return Register();
15244 if ((Subtarget->hasBWI())) {
15245 return fastEmitInst_rr(MachineInstOpcode: X86::VPSUBUSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
15246 }
15247 return Register();
15248}
15249
15250Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15251 switch (VT.SimpleTy) {
15252 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
15253 case MVT::v32i8: return fastEmit_ISD_USUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
15254 case MVT::v64i8: return fastEmit_ISD_USUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
15255 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
15256 case MVT::v16i16: return fastEmit_ISD_USUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
15257 case MVT::v32i16: return fastEmit_ISD_USUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
15258 default: return Register();
15259 }
15260}
15261
15262// FastEmit functions for ISD::XOR.
15263
15264Register fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
15265 if (RetVT.SimpleTy != MVT::i8)
15266 return Register();
15267 if ((Subtarget->hasNDD())) {
15268 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr_ND, RC: &X86::GR8RegClass, Op0, Op1);
15269 }
15270 if ((!Subtarget->hasNDD())) {
15271 return fastEmitInst_rr(MachineInstOpcode: X86::XOR8rr, RC: &X86::GR8RegClass, Op0, Op1);
15272 }
15273 return Register();
15274}
15275
15276Register fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
15277 if (RetVT.SimpleTy != MVT::i16)
15278 return Register();
15279 if ((Subtarget->hasNDD())) {
15280 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr_ND, RC: &X86::GR16RegClass, Op0, Op1);
15281 }
15282 if ((!Subtarget->hasNDD())) {
15283 return fastEmitInst_rr(MachineInstOpcode: X86::XOR16rr, RC: &X86::GR16RegClass, Op0, Op1);
15284 }
15285 return Register();
15286}
15287
15288Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15289 if (RetVT.SimpleTy != MVT::i32)
15290 return Register();
15291 if ((Subtarget->hasNDD())) {
15292 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr_ND, RC: &X86::GR32RegClass, Op0, Op1);
15293 }
15294 if ((!Subtarget->hasNDD())) {
15295 return fastEmitInst_rr(MachineInstOpcode: X86::XOR32rr, RC: &X86::GR32RegClass, Op0, Op1);
15296 }
15297 return Register();
15298}
15299
15300Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15301 if (RetVT.SimpleTy != MVT::i64)
15302 return Register();
15303 if ((Subtarget->hasNDD())) {
15304 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr_ND, RC: &X86::GR64RegClass, Op0, Op1);
15305 }
15306 if ((!Subtarget->hasNDD())) {
15307 return fastEmitInst_rr(MachineInstOpcode: X86::XOR64rr, RC: &X86::GR64RegClass, Op0, Op1);
15308 }
15309 return Register();
15310}
15311
15312Register fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
15313 if (RetVT.SimpleTy != MVT::v8i1)
15314 return Register();
15315 if ((Subtarget->hasDQI())) {
15316 return fastEmitInst_rr(MachineInstOpcode: X86::KXORBkk, RC: &X86::VK8RegClass, Op0, Op1);
15317 }
15318 return Register();
15319}
15320
15321Register fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
15322 if (RetVT.SimpleTy != MVT::v16i1)
15323 return Register();
15324 if ((Subtarget->hasAVX512())) {
15325 return fastEmitInst_rr(MachineInstOpcode: X86::KXORWkk, RC: &X86::VK16RegClass, Op0, Op1);
15326 }
15327 return Register();
15328}
15329
15330Register fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
15331 if (RetVT.SimpleTy != MVT::v32i1)
15332 return Register();
15333 if ((Subtarget->hasBWI())) {
15334 return fastEmitInst_rr(MachineInstOpcode: X86::KXORDkk, RC: &X86::VK32RegClass, Op0, Op1);
15335 }
15336 return Register();
15337}
15338
15339Register fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
15340 if (RetVT.SimpleTy != MVT::v64i1)
15341 return Register();
15342 if ((Subtarget->hasBWI())) {
15343 return fastEmitInst_rr(MachineInstOpcode: X86::KXORQkk, RC: &X86::VK64RegClass, Op0, Op1);
15344 }
15345 return Register();
15346}
15347
15348Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
15349 if (RetVT.SimpleTy != MVT::v16i8)
15350 return Register();
15351 if ((Subtarget->hasVLX())) {
15352 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15353 }
15354 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15355 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15356 }
15357 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15358 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15359 }
15360 return Register();
15361}
15362
15363Register fastEmit_ISD_XOR_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
15364 if (RetVT.SimpleTy != MVT::v32i8)
15365 return Register();
15366 if ((Subtarget->hasVLX())) {
15367 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15368 }
15369 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15370 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15371 }
15372 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15373 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
15374 }
15375 return Register();
15376}
15377
15378Register fastEmit_ISD_XOR_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
15379 if (RetVT.SimpleTy != MVT::v64i8)
15380 return Register();
15381 if ((Subtarget->hasAVX512())) {
15382 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15383 }
15384 return Register();
15385}
15386
15387Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
15388 if (RetVT.SimpleTy != MVT::v8i16)
15389 return Register();
15390 if ((Subtarget->hasVLX())) {
15391 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15392 }
15393 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15394 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15395 }
15396 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15397 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15398 }
15399 return Register();
15400}
15401
15402Register fastEmit_ISD_XOR_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
15403 if (RetVT.SimpleTy != MVT::v16i16)
15404 return Register();
15405 if ((Subtarget->hasVLX())) {
15406 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15407 }
15408 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15409 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15410 }
15411 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15412 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
15413 }
15414 return Register();
15415}
15416
15417Register fastEmit_ISD_XOR_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
15418 if (RetVT.SimpleTy != MVT::v32i16)
15419 return Register();
15420 if ((Subtarget->hasAVX512())) {
15421 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15422 }
15423 return Register();
15424}
15425
15426Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
15427 if (RetVT.SimpleTy != MVT::v4i32)
15428 return Register();
15429 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15430 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15431 }
15432 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15433 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15434 }
15435 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15436 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15437 }
15438 return Register();
15439}
15440
15441Register fastEmit_ISD_XOR_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
15442 if (RetVT.SimpleTy != MVT::v8i32)
15443 return Register();
15444 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15445 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15446 }
15447 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15448 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
15449 }
15450 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15451 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15452 }
15453 return Register();
15454}
15455
15456Register fastEmit_ISD_XOR_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
15457 if (RetVT.SimpleTy != MVT::v16i32)
15458 return Register();
15459 if ((Subtarget->hasAVX512())) {
15460 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15461 }
15462 return Register();
15463}
15464
15465Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
15466 if (RetVT.SimpleTy != MVT::v2i64)
15467 return Register();
15468 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15469 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15470 }
15471 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15472 return fastEmitInst_rr(MachineInstOpcode: X86::PXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15473 }
15474 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15475 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORrr, RC: &X86::VR128RegClass, Op0, Op1);
15476 }
15477 return Register();
15478}
15479
15480Register fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
15481 if (RetVT.SimpleTy != MVT::v4i64)
15482 return Register();
15483 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15484 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15485 }
15486 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15487 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15488 }
15489 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15490 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORYrr, RC: &X86::VR256RegClass, Op0, Op1);
15491 }
15492 return Register();
15493}
15494
15495Register fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
15496 if (RetVT.SimpleTy != MVT::v8i64)
15497 return Register();
15498 if ((Subtarget->hasAVX512())) {
15499 return fastEmitInst_rr(MachineInstOpcode: X86::VPXORQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15500 }
15501 return Register();
15502}
15503
15504Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15505 switch (VT.SimpleTy) {
15506 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op1);
15507 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op1);
15508 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
15509 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
15510 case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op1);
15511 case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op1);
15512 case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op1);
15513 case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op1);
15514 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
15515 case MVT::v32i8: return fastEmit_ISD_XOR_MVT_v32i8_rr(RetVT, Op0, Op1);
15516 case MVT::v64i8: return fastEmit_ISD_XOR_MVT_v64i8_rr(RetVT, Op0, Op1);
15517 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
15518 case MVT::v16i16: return fastEmit_ISD_XOR_MVT_v16i16_rr(RetVT, Op0, Op1);
15519 case MVT::v32i16: return fastEmit_ISD_XOR_MVT_v32i16_rr(RetVT, Op0, Op1);
15520 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
15521 case MVT::v8i32: return fastEmit_ISD_XOR_MVT_v8i32_rr(RetVT, Op0, Op1);
15522 case MVT::v16i32: return fastEmit_ISD_XOR_MVT_v16i32_rr(RetVT, Op0, Op1);
15523 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
15524 case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op1);
15525 case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op1);
15526 default: return Register();
15527 }
15528}
15529
15530// FastEmit functions for X86ISD::ADDSUB.
15531
15532Register fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15533 if (RetVT.SimpleTy != MVT::v4f32)
15534 return Register();
15535 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15536 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15537 }
15538 if ((Subtarget->hasAVX())) {
15539 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
15540 }
15541 return Register();
15542}
15543
15544Register fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
15545 if (RetVT.SimpleTy != MVT::v8f32)
15546 return Register();
15547 if ((Subtarget->hasAVX())) {
15548 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15549 }
15550 return Register();
15551}
15552
15553Register fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15554 if (RetVT.SimpleTy != MVT::v2f64)
15555 return Register();
15556 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
15557 return fastEmitInst_rr(MachineInstOpcode: X86::ADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15558 }
15559 if ((Subtarget->hasAVX())) {
15560 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
15561 }
15562 return Register();
15563}
15564
15565Register fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
15566 if (RetVT.SimpleTy != MVT::v4f64)
15567 return Register();
15568 if ((Subtarget->hasAVX())) {
15569 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
15570 }
15571 return Register();
15572}
15573
15574Register fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15575 switch (VT.SimpleTy) {
15576 case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
15577 case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
15578 case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
15579 case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
15580 default: return Register();
15581 }
15582}
15583
15584// FastEmit functions for X86ISD::ANDNP.
15585
15586Register fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
15587 if (RetVT.SimpleTy != MVT::v16i8)
15588 return Register();
15589 if ((Subtarget->hasVLX())) {
15590 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15591 }
15592 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15593 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15594 }
15595 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15596 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15597 }
15598 return Register();
15599}
15600
15601Register fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
15602 if (RetVT.SimpleTy != MVT::v32i8)
15603 return Register();
15604 if ((Subtarget->hasVLX())) {
15605 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15606 }
15607 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15608 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15609 }
15610 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15611 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
15612 }
15613 return Register();
15614}
15615
15616Register fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
15617 if (RetVT.SimpleTy != MVT::v64i8)
15618 return Register();
15619 if ((Subtarget->hasAVX512())) {
15620 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15621 }
15622 return Register();
15623}
15624
15625Register fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
15626 if (RetVT.SimpleTy != MVT::v8i16)
15627 return Register();
15628 if ((Subtarget->hasVLX())) {
15629 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15630 }
15631 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15632 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15633 }
15634 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15635 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15636 }
15637 return Register();
15638}
15639
15640Register fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
15641 if (RetVT.SimpleTy != MVT::v16i16)
15642 return Register();
15643 if ((Subtarget->hasVLX())) {
15644 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15645 }
15646 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15647 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15648 }
15649 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15650 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
15651 }
15652 return Register();
15653}
15654
15655Register fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
15656 if (RetVT.SimpleTy != MVT::v32i16)
15657 return Register();
15658 if ((Subtarget->hasAVX512())) {
15659 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15660 }
15661 return Register();
15662}
15663
15664Register fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
15665 if (RetVT.SimpleTy != MVT::v4i32)
15666 return Register();
15667 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15668 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15669 }
15670 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15671 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15672 }
15673 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15674 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15675 }
15676 return Register();
15677}
15678
15679Register fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
15680 if (RetVT.SimpleTy != MVT::v8i32)
15681 return Register();
15682 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15683 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15684 }
15685 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15686 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
15687 }
15688 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15689 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15690 }
15691 return Register();
15692}
15693
15694Register fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
15695 if (RetVT.SimpleTy != MVT::v16i32)
15696 return Register();
15697 if ((Subtarget->hasAVX512())) {
15698 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNDZrr, RC: &X86::VR512RegClass, Op0, Op1);
15699 }
15700 return Register();
15701}
15702
15703Register fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
15704 if (RetVT.SimpleTy != MVT::v2i64)
15705 return Register();
15706 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15707 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
15708 }
15709 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15710 return fastEmitInst_rr(MachineInstOpcode: X86::PANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15711 }
15712 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15713 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNrr, RC: &X86::VR128RegClass, Op0, Op1);
15714 }
15715 return Register();
15716}
15717
15718Register fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
15719 if (RetVT.SimpleTy != MVT::v4i64)
15720 return Register();
15721 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
15722 return fastEmitInst_rr(MachineInstOpcode: X86::VANDNPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
15723 }
15724 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15725 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
15726 }
15727 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15728 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNYrr, RC: &X86::VR256RegClass, Op0, Op1);
15729 }
15730 return Register();
15731}
15732
15733Register fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
15734 if (RetVT.SimpleTy != MVT::v8i64)
15735 return Register();
15736 if ((Subtarget->hasAVX512())) {
15737 return fastEmitInst_rr(MachineInstOpcode: X86::VPANDNQZrr, RC: &X86::VR512RegClass, Op0, Op1);
15738 }
15739 return Register();
15740}
15741
15742Register fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15743 switch (VT.SimpleTy) {
15744 case MVT::v16i8: return fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(RetVT, Op0, Op1);
15745 case MVT::v32i8: return fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(RetVT, Op0, Op1);
15746 case MVT::v64i8: return fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(RetVT, Op0, Op1);
15747 case MVT::v8i16: return fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(RetVT, Op0, Op1);
15748 case MVT::v16i16: return fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(RetVT, Op0, Op1);
15749 case MVT::v32i16: return fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(RetVT, Op0, Op1);
15750 case MVT::v4i32: return fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(RetVT, Op0, Op1);
15751 case MVT::v8i32: return fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(RetVT, Op0, Op1);
15752 case MVT::v16i32: return fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(RetVT, Op0, Op1);
15753 case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op1);
15754 case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op1);
15755 case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op1);
15756 default: return Register();
15757 }
15758}
15759
15760// FastEmit functions for X86ISD::BEXTR.
15761
15762Register fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15763 if (RetVT.SimpleTy != MVT::i32)
15764 return Register();
15765 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15766 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
15767 }
15768 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
15769 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR32rr, RC: &X86::GR32RegClass, Op0, Op1);
15770 }
15771 return Register();
15772}
15773
15774Register fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15775 if (RetVT.SimpleTy != MVT::i64)
15776 return Register();
15777 if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15778 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
15779 }
15780 if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
15781 return fastEmitInst_rr(MachineInstOpcode: X86::BEXTR64rr, RC: &X86::GR64RegClass, Op0, Op1);
15782 }
15783 return Register();
15784}
15785
15786Register fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15787 switch (VT.SimpleTy) {
15788 case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op1);
15789 case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op1);
15790 default: return Register();
15791 }
15792}
15793
15794// FastEmit functions for X86ISD::BT.
15795
15796Register fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
15797 if (RetVT.SimpleTy != MVT::i32)
15798 return Register();
15799 return fastEmitInst_rr(MachineInstOpcode: X86::BT16rr, RC: &X86::GR16RegClass, Op0, Op1);
15800}
15801
15802Register fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15803 if (RetVT.SimpleTy != MVT::i32)
15804 return Register();
15805 return fastEmitInst_rr(MachineInstOpcode: X86::BT32rr, RC: &X86::GR32RegClass, Op0, Op1);
15806}
15807
15808Register fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15809 if (RetVT.SimpleTy != MVT::i32)
15810 return Register();
15811 return fastEmitInst_rr(MachineInstOpcode: X86::BT64rr, RC: &X86::GR64RegClass, Op0, Op1);
15812}
15813
15814Register fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15815 switch (VT.SimpleTy) {
15816 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op1);
15817 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op1);
15818 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op1);
15819 default: return Register();
15820 }
15821}
15822
15823// FastEmit functions for X86ISD::BZHI.
15824
15825Register fastEmit_X86ISD_BZHI_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15826 if (RetVT.SimpleTy != MVT::i32)
15827 return Register();
15828 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15829 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr_EVEX, RC: &X86::GR32RegClass, Op0, Op1);
15830 }
15831 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15832 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI32rr, RC: &X86::GR32RegClass, Op0, Op1);
15833 }
15834 return Register();
15835}
15836
15837Register fastEmit_X86ISD_BZHI_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15838 if (RetVT.SimpleTy != MVT::i64)
15839 return Register();
15840 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
15841 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr_EVEX, RC: &X86::GR64RegClass, Op0, Op1);
15842 }
15843 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15844 return fastEmitInst_rr(MachineInstOpcode: X86::BZHI64rr, RC: &X86::GR64RegClass, Op0, Op1);
15845 }
15846 return Register();
15847}
15848
15849Register fastEmit_X86ISD_BZHI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15850 switch (VT.SimpleTy) {
15851 case MVT::i32: return fastEmit_X86ISD_BZHI_MVT_i32_rr(RetVT, Op0, Op1);
15852 case MVT::i64: return fastEmit_X86ISD_BZHI_MVT_i64_rr(RetVT, Op0, Op1);
15853 default: return Register();
15854 }
15855}
15856
15857// FastEmit functions for X86ISD::CMP.
15858
15859Register fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, Register Op0, Register Op1) {
15860 if (RetVT.SimpleTy != MVT::i32)
15861 return Register();
15862 return fastEmitInst_rr(MachineInstOpcode: X86::CMP8rr, RC: &X86::GR8RegClass, Op0, Op1);
15863}
15864
15865Register fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, Register Op0, Register Op1) {
15866 if (RetVT.SimpleTy != MVT::i32)
15867 return Register();
15868 return fastEmitInst_rr(MachineInstOpcode: X86::CMP16rr, RC: &X86::GR16RegClass, Op0, Op1);
15869}
15870
15871Register fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
15872 if (RetVT.SimpleTy != MVT::i32)
15873 return Register();
15874 return fastEmitInst_rr(MachineInstOpcode: X86::CMP32rr, RC: &X86::GR32RegClass, Op0, Op1);
15875}
15876
15877Register fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
15878 if (RetVT.SimpleTy != MVT::i32)
15879 return Register();
15880 return fastEmitInst_rr(MachineInstOpcode: X86::CMP64rr, RC: &X86::GR64RegClass, Op0, Op1);
15881}
15882
15883Register fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15884 switch (VT.SimpleTy) {
15885 case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op1);
15886 case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op1);
15887 case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
15888 case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op1);
15889 default: return Register();
15890 }
15891}
15892
15893// FastEmit functions for X86ISD::COMI.
15894
15895Register fastEmit_X86ISD_COMI_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15896 if (RetVT.SimpleTy != MVT::i32)
15897 return Register();
15898 if ((Subtarget->hasFP16())) {
15899 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15900 }
15901 return Register();
15902}
15903
15904Register fastEmit_X86ISD_COMI_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
15905 if (RetVT.SimpleTy != MVT::i32)
15906 return Register();
15907 if ((Subtarget->hasAVX10_2())) {
15908 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISBF16Zrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15909 }
15910 return Register();
15911}
15912
15913Register fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15914 if (RetVT.SimpleTy != MVT::i32)
15915 return Register();
15916 if ((Subtarget->hasAVX512())) {
15917 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15918 }
15919 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15920 return fastEmitInst_rr(MachineInstOpcode: X86::COMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15921 }
15922 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15923 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15924 }
15925 return Register();
15926}
15927
15928Register fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15929 if (RetVT.SimpleTy != MVT::i32)
15930 return Register();
15931 if ((Subtarget->hasAVX512())) {
15932 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15933 }
15934 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15935 return fastEmitInst_rr(MachineInstOpcode: X86::COMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15936 }
15937 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15938 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
15939 }
15940 return Register();
15941}
15942
15943Register fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15944 switch (VT.SimpleTy) {
15945 case MVT::v8f16: return fastEmit_X86ISD_COMI_MVT_v8f16_rr(RetVT, Op0, Op1);
15946 case MVT::v8bf16: return fastEmit_X86ISD_COMI_MVT_v8bf16_rr(RetVT, Op0, Op1);
15947 case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op1);
15948 case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op1);
15949 default: return Register();
15950 }
15951}
15952
15953// FastEmit functions for X86ISD::COMX.
15954
15955Register fastEmit_X86ISD_COMX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15956 if (RetVT.SimpleTy != MVT::i32)
15957 return Register();
15958 if ((Subtarget->hasAVX10_2())) {
15959 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15960 }
15961 return Register();
15962}
15963
15964Register fastEmit_X86ISD_COMX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
15965 if (RetVT.SimpleTy != MVT::i32)
15966 return Register();
15967 if ((Subtarget->hasAVX10_2())) {
15968 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15969 }
15970 return Register();
15971}
15972
15973Register fastEmit_X86ISD_COMX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
15974 if (RetVT.SimpleTy != MVT::i32)
15975 return Register();
15976 if ((Subtarget->hasAVX10_2())) {
15977 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15978 }
15979 return Register();
15980}
15981
15982Register fastEmit_X86ISD_COMX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
15983 switch (VT.SimpleTy) {
15984 case MVT::v8f16: return fastEmit_X86ISD_COMX_MVT_v8f16_rr(RetVT, Op0, Op1);
15985 case MVT::v4f32: return fastEmit_X86ISD_COMX_MVT_v4f32_rr(RetVT, Op0, Op1);
15986 case MVT::v2f64: return fastEmit_X86ISD_COMX_MVT_v2f64_rr(RetVT, Op0, Op1);
15987 default: return Register();
15988 }
15989}
15990
15991// FastEmit functions for X86ISD::FADDS.
15992
15993Register fastEmit_X86ISD_FADDS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
15994 if (RetVT.SimpleTy != MVT::v8f16)
15995 return Register();
15996 if ((Subtarget->hasFP16())) {
15997 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
15998 }
15999 return Register();
16000}
16001
16002Register fastEmit_X86ISD_FADDS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16003 if (RetVT.SimpleTy != MVT::v4f32)
16004 return Register();
16005 if ((Subtarget->hasAVX512())) {
16006 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16007 }
16008 return Register();
16009}
16010
16011Register fastEmit_X86ISD_FADDS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16012 if (RetVT.SimpleTy != MVT::v2f64)
16013 return Register();
16014 if ((Subtarget->hasAVX512())) {
16015 return fastEmitInst_rr(MachineInstOpcode: X86::VADDSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16016 }
16017 return Register();
16018}
16019
16020Register fastEmit_X86ISD_FADDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16021 switch (VT.SimpleTy) {
16022 case MVT::v8f16: return fastEmit_X86ISD_FADDS_MVT_v8f16_rr(RetVT, Op0, Op1);
16023 case MVT::v4f32: return fastEmit_X86ISD_FADDS_MVT_v4f32_rr(RetVT, Op0, Op1);
16024 case MVT::v2f64: return fastEmit_X86ISD_FADDS_MVT_v2f64_rr(RetVT, Op0, Op1);
16025 default: return Register();
16026 }
16027}
16028
16029// FastEmit functions for X86ISD::FAND.
16030
16031Register fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
16032 if (RetVT.SimpleTy != MVT::f128)
16033 return Register();
16034 if ((Subtarget->hasVLX())) {
16035 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16036 }
16037 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16038 return fastEmitInst_rr(MachineInstOpcode: X86::VANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16039 }
16040 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16041 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16042 }
16043 return Register();
16044}
16045
16046Register fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16047 if (RetVT.SimpleTy != MVT::v4f32)
16048 return Register();
16049 return fastEmitInst_rr(MachineInstOpcode: X86::ANDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16050}
16051
16052Register fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16053 switch (VT.SimpleTy) {
16054 case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op1);
16055 case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op1);
16056 default: return Register();
16057 }
16058}
16059
16060// FastEmit functions for X86ISD::FANDN.
16061
16062Register fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16063 if (RetVT.SimpleTy != MVT::v4f32)
16064 return Register();
16065 return fastEmitInst_rr(MachineInstOpcode: X86::ANDNPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16066}
16067
16068Register fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16069 switch (VT.SimpleTy) {
16070 case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op1);
16071 default: return Register();
16072 }
16073}
16074
16075// FastEmit functions for X86ISD::FCMP.
16076
16077Register fastEmit_X86ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16078 if (RetVT.SimpleTy != MVT::i32)
16079 return Register();
16080 if ((Subtarget->hasFP16())) {
16081 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16082 }
16083 return Register();
16084}
16085
16086Register fastEmit_X86ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16087 if (RetVT.SimpleTy != MVT::i32)
16088 return Register();
16089 if ((Subtarget->hasAVX512())) {
16090 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16091 }
16092 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16093 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
16094 }
16095 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16096 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
16097 }
16098 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
16099 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
16100 }
16101 return Register();
16102}
16103
16104Register fastEmit_X86ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16105 if (RetVT.SimpleTy != MVT::i32)
16106 return Register();
16107 if ((Subtarget->hasAVX512())) {
16108 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16109 }
16110 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16111 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
16112 }
16113 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16114 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
16115 }
16116 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
16117 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
16118 }
16119 return Register();
16120}
16121
16122Register fastEmit_X86ISD_FCMP_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
16123 if (RetVT.SimpleTy != MVT::i32)
16124 return Register();
16125 if ((Subtarget->canUseCMOV())) {
16126 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
16127 }
16128 return Register();
16129}
16130
16131Register fastEmit_X86ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16132 switch (VT.SimpleTy) {
16133 case MVT::f16: return fastEmit_X86ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
16134 case MVT::f32: return fastEmit_X86ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
16135 case MVT::f64: return fastEmit_X86ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
16136 case MVT::f80: return fastEmit_X86ISD_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
16137 default: return Register();
16138 }
16139}
16140
16141// FastEmit functions for X86ISD::FDIVS.
16142
16143Register fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16144 if (RetVT.SimpleTy != MVT::v8f16)
16145 return Register();
16146 if ((Subtarget->hasFP16())) {
16147 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16148 }
16149 return Register();
16150}
16151
16152Register fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16153 if (RetVT.SimpleTy != MVT::v4f32)
16154 return Register();
16155 if ((Subtarget->hasAVX512())) {
16156 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16157 }
16158 return Register();
16159}
16160
16161Register fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16162 if (RetVT.SimpleTy != MVT::v2f64)
16163 return Register();
16164 if ((Subtarget->hasAVX512())) {
16165 return fastEmitInst_rr(MachineInstOpcode: X86::VDIVSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16166 }
16167 return Register();
16168}
16169
16170Register fastEmit_X86ISD_FDIVS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16171 switch (VT.SimpleTy) {
16172 case MVT::v8f16: return fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(RetVT, Op0, Op1);
16173 case MVT::v4f32: return fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(RetVT, Op0, Op1);
16174 case MVT::v2f64: return fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(RetVT, Op0, Op1);
16175 default: return Register();
16176 }
16177}
16178
16179// FastEmit functions for X86ISD::FGETEXPS.
16180
16181Register fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16182 if (RetVT.SimpleTy != MVT::v8f16)
16183 return Register();
16184 if ((Subtarget->hasFP16())) {
16185 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZr, RC: &X86::VR128XRegClass, Op0, Op1);
16186 }
16187 return Register();
16188}
16189
16190Register fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16191 if (RetVT.SimpleTy != MVT::v4f32)
16192 return Register();
16193 if ((Subtarget->hasAVX512())) {
16194 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZr, RC: &X86::VR128XRegClass, Op0, Op1);
16195 }
16196 return Register();
16197}
16198
16199Register fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16200 if (RetVT.SimpleTy != MVT::v2f64)
16201 return Register();
16202 if ((Subtarget->hasAVX512())) {
16203 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZr, RC: &X86::VR128XRegClass, Op0, Op1);
16204 }
16205 return Register();
16206}
16207
16208Register fastEmit_X86ISD_FGETEXPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16209 switch (VT.SimpleTy) {
16210 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(RetVT, Op0, Op1);
16211 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(RetVT, Op0, Op1);
16212 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(RetVT, Op0, Op1);
16213 default: return Register();
16214 }
16215}
16216
16217// FastEmit functions for X86ISD::FGETEXPS_SAE.
16218
16219Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16220 if (RetVT.SimpleTy != MVT::v8f16)
16221 return Register();
16222 if ((Subtarget->hasFP16())) {
16223 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSHZrb, RC: &X86::VR128XRegClass, Op0, Op1);
16224 }
16225 return Register();
16226}
16227
16228Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16229 if (RetVT.SimpleTy != MVT::v4f32)
16230 return Register();
16231 if ((Subtarget->hasAVX512())) {
16232 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSSZrb, RC: &X86::VR128XRegClass, Op0, Op1);
16233 }
16234 return Register();
16235}
16236
16237Register fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16238 if (RetVT.SimpleTy != MVT::v2f64)
16239 return Register();
16240 if ((Subtarget->hasAVX512())) {
16241 return fastEmitInst_rr(MachineInstOpcode: X86::VGETEXPSDZrb, RC: &X86::VR128XRegClass, Op0, Op1);
16242 }
16243 return Register();
16244}
16245
16246Register fastEmit_X86ISD_FGETEXPS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16247 switch (VT.SimpleTy) {
16248 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
16249 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16250 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16251 default: return Register();
16252 }
16253}
16254
16255// FastEmit functions for X86ISD::FHADD.
16256
16257Register fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16258 if (RetVT.SimpleTy != MVT::v4f32)
16259 return Register();
16260 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
16261 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16262 }
16263 if ((Subtarget->hasAVX())) {
16264 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16265 }
16266 return Register();
16267}
16268
16269Register fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16270 if (RetVT.SimpleTy != MVT::v8f32)
16271 return Register();
16272 if ((Subtarget->hasAVX())) {
16273 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16274 }
16275 return Register();
16276}
16277
16278Register fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16279 if (RetVT.SimpleTy != MVT::v2f64)
16280 return Register();
16281 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
16282 return fastEmitInst_rr(MachineInstOpcode: X86::HADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16283 }
16284 if ((Subtarget->hasAVX())) {
16285 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16286 }
16287 return Register();
16288}
16289
16290Register fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16291 if (RetVT.SimpleTy != MVT::v4f64)
16292 return Register();
16293 if ((Subtarget->hasAVX())) {
16294 return fastEmitInst_rr(MachineInstOpcode: X86::VHADDPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16295 }
16296 return Register();
16297}
16298
16299Register fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16300 switch (VT.SimpleTy) {
16301 case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op1);
16302 case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op1);
16303 case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op1);
16304 case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op1);
16305 default: return Register();
16306 }
16307}
16308
16309// FastEmit functions for X86ISD::FHSUB.
16310
16311Register fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16312 if (RetVT.SimpleTy != MVT::v4f32)
16313 return Register();
16314 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
16315 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16316 }
16317 if ((Subtarget->hasAVX())) {
16318 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16319 }
16320 return Register();
16321}
16322
16323Register fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16324 if (RetVT.SimpleTy != MVT::v8f32)
16325 return Register();
16326 if ((Subtarget->hasAVX())) {
16327 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16328 }
16329 return Register();
16330}
16331
16332Register fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16333 if (RetVT.SimpleTy != MVT::v2f64)
16334 return Register();
16335 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
16336 return fastEmitInst_rr(MachineInstOpcode: X86::HSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16337 }
16338 if ((Subtarget->hasAVX())) {
16339 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16340 }
16341 return Register();
16342}
16343
16344Register fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16345 if (RetVT.SimpleTy != MVT::v4f64)
16346 return Register();
16347 if ((Subtarget->hasAVX())) {
16348 return fastEmitInst_rr(MachineInstOpcode: X86::VHSUBPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16349 }
16350 return Register();
16351}
16352
16353Register fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16354 switch (VT.SimpleTy) {
16355 case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
16356 case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
16357 case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
16358 case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
16359 default: return Register();
16360 }
16361}
16362
16363// FastEmit functions for X86ISD::FMAXC.
16364
16365Register fastEmit_X86ISD_FMAXC_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16366 if (RetVT.SimpleTy != MVT::f16)
16367 return Register();
16368 if ((Subtarget->hasAVX512())) {
16369 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16370 }
16371 return Register();
16372}
16373
16374Register fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16375 if (RetVT.SimpleTy != MVT::f32)
16376 return Register();
16377 if ((Subtarget->hasAVX512())) {
16378 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16379 }
16380 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16381 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16382 }
16383 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16384 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16385 }
16386 return Register();
16387}
16388
16389Register fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16390 if (RetVT.SimpleTy != MVT::f64)
16391 return Register();
16392 if ((Subtarget->hasAVX512())) {
16393 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16394 }
16395 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16396 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16397 }
16398 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16399 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16400 }
16401 return Register();
16402}
16403
16404Register fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16405 if (RetVT.SimpleTy != MVT::v8f16)
16406 return Register();
16407 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16408 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16409 }
16410 return Register();
16411}
16412
16413Register fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
16414 if (RetVT.SimpleTy != MVT::v16f16)
16415 return Register();
16416 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16417 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16418 }
16419 return Register();
16420}
16421
16422Register fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16423 if (RetVT.SimpleTy != MVT::v32f16)
16424 return Register();
16425 if ((Subtarget->hasFP16())) {
16426 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16427 }
16428 return Register();
16429}
16430
16431Register fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16432 if (RetVT.SimpleTy != MVT::v4f32)
16433 return Register();
16434 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16435 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16436 }
16437 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16438 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16439 }
16440 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16441 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16442 }
16443 return Register();
16444}
16445
16446Register fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16447 if (RetVT.SimpleTy != MVT::v8f32)
16448 return Register();
16449 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16450 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16451 }
16452 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16453 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16454 }
16455 return Register();
16456}
16457
16458Register fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16459 if (RetVT.SimpleTy != MVT::v16f32)
16460 return Register();
16461 if ((Subtarget->hasAVX512())) {
16462 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16463 }
16464 return Register();
16465}
16466
16467Register fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16468 if (RetVT.SimpleTy != MVT::v2f64)
16469 return Register();
16470 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16471 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16472 }
16473 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16474 return fastEmitInst_rr(MachineInstOpcode: X86::MAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16475 }
16476 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16477 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16478 }
16479 return Register();
16480}
16481
16482Register fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16483 if (RetVT.SimpleTy != MVT::v4f64)
16484 return Register();
16485 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16486 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16487 }
16488 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16489 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16490 }
16491 return Register();
16492}
16493
16494Register fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16495 if (RetVT.SimpleTy != MVT::v8f64)
16496 return Register();
16497 if ((Subtarget->hasAVX512())) {
16498 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16499 }
16500 return Register();
16501}
16502
16503Register fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16504 switch (VT.SimpleTy) {
16505 case MVT::f16: return fastEmit_X86ISD_FMAXC_MVT_f16_rr(RetVT, Op0, Op1);
16506 case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op1);
16507 case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op1);
16508 case MVT::v8f16: return fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(RetVT, Op0, Op1);
16509 case MVT::v16f16: return fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(RetVT, Op0, Op1);
16510 case MVT::v32f16: return fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(RetVT, Op0, Op1);
16511 case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op1);
16512 case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op1);
16513 case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op1);
16514 case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op1);
16515 case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op1);
16516 case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op1);
16517 default: return Register();
16518 }
16519}
16520
16521// FastEmit functions for X86ISD::FMAXS.
16522
16523Register fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16524 if (RetVT.SimpleTy != MVT::v8f16)
16525 return Register();
16526 if ((Subtarget->hasFP16())) {
16527 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16528 }
16529 return Register();
16530}
16531
16532Register fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16533 if (RetVT.SimpleTy != MVT::v4f32)
16534 return Register();
16535 if ((Subtarget->hasAVX512())) {
16536 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16537 }
16538 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16539 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16540 }
16541 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16542 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16543 }
16544 return Register();
16545}
16546
16547Register fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16548 if (RetVT.SimpleTy != MVT::v2f64)
16549 return Register();
16550 if ((Subtarget->hasAVX512())) {
16551 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16552 }
16553 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16554 return fastEmitInst_rr(MachineInstOpcode: X86::MAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16555 }
16556 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16557 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16558 }
16559 return Register();
16560}
16561
16562Register fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16563 switch (VT.SimpleTy) {
16564 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(RetVT, Op0, Op1);
16565 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op1);
16566 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op1);
16567 default: return Register();
16568 }
16569}
16570
16571// FastEmit functions for X86ISD::FMAXS_SAE.
16572
16573Register fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16574 if (RetVT.SimpleTy != MVT::v8f16)
16575 return Register();
16576 if ((Subtarget->hasFP16())) {
16577 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16578 }
16579 return Register();
16580}
16581
16582Register fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16583 if (RetVT.SimpleTy != MVT::v4f32)
16584 return Register();
16585 if ((Subtarget->hasAVX512())) {
16586 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16587 }
16588 return Register();
16589}
16590
16591Register fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16592 if (RetVT.SimpleTy != MVT::v2f64)
16593 return Register();
16594 if ((Subtarget->hasAVX512())) {
16595 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16596 }
16597 return Register();
16598}
16599
16600Register fastEmit_X86ISD_FMAXS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16601 switch (VT.SimpleTy) {
16602 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
16603 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16604 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16605 default: return Register();
16606 }
16607}
16608
16609// FastEmit functions for X86ISD::FMAX_SAE.
16610
16611Register fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16612 if (RetVT.SimpleTy != MVT::v32f16)
16613 return Register();
16614 if ((Subtarget->hasFP16())) {
16615 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16616 }
16617 return Register();
16618}
16619
16620Register fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16621 if (RetVT.SimpleTy != MVT::v16f32)
16622 return Register();
16623 if ((Subtarget->hasAVX512())) {
16624 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16625 }
16626 return Register();
16627}
16628
16629Register fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16630 if (RetVT.SimpleTy != MVT::v8f64)
16631 return Register();
16632 if ((Subtarget->hasAVX512())) {
16633 return fastEmitInst_rr(MachineInstOpcode: X86::VMAXPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16634 }
16635 return Register();
16636}
16637
16638Register fastEmit_X86ISD_FMAX_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16639 switch (VT.SimpleTy) {
16640 case MVT::v32f16: return fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
16641 case MVT::v16f32: return fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
16642 case MVT::v8f64: return fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
16643 default: return Register();
16644 }
16645}
16646
16647// FastEmit functions for X86ISD::FMINC.
16648
16649Register fastEmit_X86ISD_FMINC_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
16650 if (RetVT.SimpleTy != MVT::f16)
16651 return Register();
16652 if ((Subtarget->hasAVX512())) {
16653 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
16654 }
16655 return Register();
16656}
16657
16658Register fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
16659 if (RetVT.SimpleTy != MVT::f32)
16660 return Register();
16661 if ((Subtarget->hasAVX512())) {
16662 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
16663 }
16664 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16665 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16666 }
16667 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16668 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSSrr, RC: &X86::FR32RegClass, Op0, Op1);
16669 }
16670 return Register();
16671}
16672
16673Register fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
16674 if (RetVT.SimpleTy != MVT::f64)
16675 return Register();
16676 if ((Subtarget->hasAVX512())) {
16677 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
16678 }
16679 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16680 return fastEmitInst_rr(MachineInstOpcode: X86::MINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16681 }
16682 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16683 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCSDrr, RC: &X86::FR64RegClass, Op0, Op1);
16684 }
16685 return Register();
16686}
16687
16688Register fastEmit_X86ISD_FMINC_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16689 if (RetVT.SimpleTy != MVT::v8f16)
16690 return Register();
16691 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16692 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16693 }
16694 return Register();
16695}
16696
16697Register fastEmit_X86ISD_FMINC_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
16698 if (RetVT.SimpleTy != MVT::v16f16)
16699 return Register();
16700 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16701 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16702 }
16703 return Register();
16704}
16705
16706Register fastEmit_X86ISD_FMINC_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16707 if (RetVT.SimpleTy != MVT::v32f16)
16708 return Register();
16709 if ((Subtarget->hasFP16())) {
16710 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
16711 }
16712 return Register();
16713}
16714
16715Register fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16716 if (RetVT.SimpleTy != MVT::v4f32)
16717 return Register();
16718 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16719 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16720 }
16721 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16722 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16723 }
16724 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16725 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16726 }
16727 return Register();
16728}
16729
16730Register fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
16731 if (RetVT.SimpleTy != MVT::v8f32)
16732 return Register();
16733 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16734 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16735 }
16736 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16737 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
16738 }
16739 return Register();
16740}
16741
16742Register fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16743 if (RetVT.SimpleTy != MVT::v16f32)
16744 return Register();
16745 if ((Subtarget->hasAVX512())) {
16746 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
16747 }
16748 return Register();
16749}
16750
16751Register fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16752 if (RetVT.SimpleTy != MVT::v2f64)
16753 return Register();
16754 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16755 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16756 }
16757 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16758 return fastEmitInst_rr(MachineInstOpcode: X86::MINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16759 }
16760 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16761 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDrr, RC: &X86::VR128RegClass, Op0, Op1);
16762 }
16763 return Register();
16764}
16765
16766Register fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
16767 if (RetVT.SimpleTy != MVT::v4f64)
16768 return Register();
16769 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16770 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
16771 }
16772 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16773 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
16774 }
16775 return Register();
16776}
16777
16778Register fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16779 if (RetVT.SimpleTy != MVT::v8f64)
16780 return Register();
16781 if ((Subtarget->hasAVX512())) {
16782 return fastEmitInst_rr(MachineInstOpcode: X86::VMINCPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
16783 }
16784 return Register();
16785}
16786
16787Register fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16788 switch (VT.SimpleTy) {
16789 case MVT::f16: return fastEmit_X86ISD_FMINC_MVT_f16_rr(RetVT, Op0, Op1);
16790 case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op1);
16791 case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op1);
16792 case MVT::v8f16: return fastEmit_X86ISD_FMINC_MVT_v8f16_rr(RetVT, Op0, Op1);
16793 case MVT::v16f16: return fastEmit_X86ISD_FMINC_MVT_v16f16_rr(RetVT, Op0, Op1);
16794 case MVT::v32f16: return fastEmit_X86ISD_FMINC_MVT_v32f16_rr(RetVT, Op0, Op1);
16795 case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op1);
16796 case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op1);
16797 case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op1);
16798 case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op1);
16799 case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op1);
16800 case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op1);
16801 default: return Register();
16802 }
16803}
16804
16805// FastEmit functions for X86ISD::FMINS.
16806
16807Register fastEmit_X86ISD_FMINS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16808 if (RetVT.SimpleTy != MVT::v8f16)
16809 return Register();
16810 if ((Subtarget->hasFP16())) {
16811 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16812 }
16813 return Register();
16814}
16815
16816Register fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16817 if (RetVT.SimpleTy != MVT::v4f32)
16818 return Register();
16819 if ((Subtarget->hasAVX512())) {
16820 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16821 }
16822 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16823 return fastEmitInst_rr(MachineInstOpcode: X86::MINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16824 }
16825 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16826 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16827 }
16828 return Register();
16829}
16830
16831Register fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16832 if (RetVT.SimpleTy != MVT::v2f64)
16833 return Register();
16834 if ((Subtarget->hasAVX512())) {
16835 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16836 }
16837 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16838 return fastEmitInst_rr(MachineInstOpcode: X86::MINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16839 }
16840 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16841 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
16842 }
16843 return Register();
16844}
16845
16846Register fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16847 switch (VT.SimpleTy) {
16848 case MVT::v8f16: return fastEmit_X86ISD_FMINS_MVT_v8f16_rr(RetVT, Op0, Op1);
16849 case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op1);
16850 case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op1);
16851 default: return Register();
16852 }
16853}
16854
16855// FastEmit functions for X86ISD::FMINS_SAE.
16856
16857Register fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16858 if (RetVT.SimpleTy != MVT::v8f16)
16859 return Register();
16860 if ((Subtarget->hasFP16())) {
16861 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSHZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16862 }
16863 return Register();
16864}
16865
16866Register fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16867 if (RetVT.SimpleTy != MVT::v4f32)
16868 return Register();
16869 if ((Subtarget->hasAVX512())) {
16870 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSSZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16871 }
16872 return Register();
16873}
16874
16875Register fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16876 if (RetVT.SimpleTy != MVT::v2f64)
16877 return Register();
16878 if ((Subtarget->hasAVX512())) {
16879 return fastEmitInst_rr(MachineInstOpcode: X86::VMINSDZrrb_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16880 }
16881 return Register();
16882}
16883
16884Register fastEmit_X86ISD_FMINS_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16885 switch (VT.SimpleTy) {
16886 case MVT::v8f16: return fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
16887 case MVT::v4f32: return fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16888 case MVT::v2f64: return fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16889 default: return Register();
16890 }
16891}
16892
16893// FastEmit functions for X86ISD::FMIN_SAE.
16894
16895Register fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
16896 if (RetVT.SimpleTy != MVT::v32f16)
16897 return Register();
16898 if ((Subtarget->hasFP16())) {
16899 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPHZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16900 }
16901 return Register();
16902}
16903
16904Register fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
16905 if (RetVT.SimpleTy != MVT::v16f32)
16906 return Register();
16907 if ((Subtarget->hasAVX512())) {
16908 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPSZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16909 }
16910 return Register();
16911}
16912
16913Register fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
16914 if (RetVT.SimpleTy != MVT::v8f64)
16915 return Register();
16916 if ((Subtarget->hasAVX512())) {
16917 return fastEmitInst_rr(MachineInstOpcode: X86::VMINPDZrrb, RC: &X86::VR512RegClass, Op0, Op1);
16918 }
16919 return Register();
16920}
16921
16922Register fastEmit_X86ISD_FMIN_SAE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16923 switch (VT.SimpleTy) {
16924 case MVT::v32f16: return fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
16925 case MVT::v16f32: return fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
16926 case MVT::v8f64: return fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
16927 default: return Register();
16928 }
16929}
16930
16931// FastEmit functions for X86ISD::FMULS.
16932
16933Register fastEmit_X86ISD_FMULS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
16934 if (RetVT.SimpleTy != MVT::v8f16)
16935 return Register();
16936 if ((Subtarget->hasFP16())) {
16937 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16938 }
16939 return Register();
16940}
16941
16942Register fastEmit_X86ISD_FMULS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16943 if (RetVT.SimpleTy != MVT::v4f32)
16944 return Register();
16945 if ((Subtarget->hasAVX512())) {
16946 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16947 }
16948 return Register();
16949}
16950
16951Register fastEmit_X86ISD_FMULS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
16952 if (RetVT.SimpleTy != MVT::v2f64)
16953 return Register();
16954 if ((Subtarget->hasAVX512())) {
16955 return fastEmitInst_rr(MachineInstOpcode: X86::VMULSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
16956 }
16957 return Register();
16958}
16959
16960Register fastEmit_X86ISD_FMULS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16961 switch (VT.SimpleTy) {
16962 case MVT::v8f16: return fastEmit_X86ISD_FMULS_MVT_v8f16_rr(RetVT, Op0, Op1);
16963 case MVT::v4f32: return fastEmit_X86ISD_FMULS_MVT_v4f32_rr(RetVT, Op0, Op1);
16964 case MVT::v2f64: return fastEmit_X86ISD_FMULS_MVT_v2f64_rr(RetVT, Op0, Op1);
16965 default: return Register();
16966 }
16967}
16968
16969// FastEmit functions for X86ISD::FOR.
16970
16971Register fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
16972 if (RetVT.SimpleTy != MVT::f128)
16973 return Register();
16974 if ((Subtarget->hasVLX())) {
16975 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
16976 }
16977 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16978 return fastEmitInst_rr(MachineInstOpcode: X86::VORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16979 }
16980 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16981 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16982 }
16983 return Register();
16984}
16985
16986Register fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
16987 if (RetVT.SimpleTy != MVT::v4f32)
16988 return Register();
16989 return fastEmitInst_rr(MachineInstOpcode: X86::ORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
16990}
16991
16992Register fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
16993 switch (VT.SimpleTy) {
16994 case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op1);
16995 case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op1);
16996 default: return Register();
16997 }
16998}
16999
17000// FastEmit functions for X86ISD::FP80_ADD.
17001
17002Register fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
17003 if (RetVT.SimpleTy != MVT::f80)
17004 return Register();
17005 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
17006}
17007
17008Register fastEmit_X86ISD_FP80_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17009 switch (VT.SimpleTy) {
17010 case MVT::f80: return fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
17011 default: return Register();
17012 }
17013}
17014
17015// FastEmit functions for X86ISD::FSQRTS.
17016
17017Register fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17018 if (RetVT.SimpleTy != MVT::v8f16)
17019 return Register();
17020 if ((Subtarget->hasFP16())) {
17021 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSHZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
17022 }
17023 return Register();
17024}
17025
17026Register fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17027 if (RetVT.SimpleTy != MVT::v4f32)
17028 return Register();
17029 if ((Subtarget->hasAVX512())) {
17030 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSSZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
17031 }
17032 return Register();
17033}
17034
17035Register fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17036 if (RetVT.SimpleTy != MVT::v2f64)
17037 return Register();
17038 if ((Subtarget->hasAVX512())) {
17039 return fastEmitInst_rr(MachineInstOpcode: X86::VSQRTSDZr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
17040 }
17041 return Register();
17042}
17043
17044Register fastEmit_X86ISD_FSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17045 switch (VT.SimpleTy) {
17046 case MVT::v8f16: return fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(RetVT, Op0, Op1);
17047 case MVT::v4f32: return fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
17048 case MVT::v2f64: return fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
17049 default: return Register();
17050 }
17051}
17052
17053// FastEmit functions for X86ISD::FSUBS.
17054
17055Register fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17056 if (RetVT.SimpleTy != MVT::v8f16)
17057 return Register();
17058 if ((Subtarget->hasFP16())) {
17059 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
17060 }
17061 return Register();
17062}
17063
17064Register fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17065 if (RetVT.SimpleTy != MVT::v4f32)
17066 return Register();
17067 if ((Subtarget->hasAVX512())) {
17068 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
17069 }
17070 return Register();
17071}
17072
17073Register fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17074 if (RetVT.SimpleTy != MVT::v2f64)
17075 return Register();
17076 if ((Subtarget->hasAVX512())) {
17077 return fastEmitInst_rr(MachineInstOpcode: X86::VSUBSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
17078 }
17079 return Register();
17080}
17081
17082Register fastEmit_X86ISD_FSUBS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17083 switch (VT.SimpleTy) {
17084 case MVT::v8f16: return fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(RetVT, Op0, Op1);
17085 case MVT::v4f32: return fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(RetVT, Op0, Op1);
17086 case MVT::v2f64: return fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(RetVT, Op0, Op1);
17087 default: return Register();
17088 }
17089}
17090
17091// FastEmit functions for X86ISD::FXOR.
17092
17093Register fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, Register Op0, Register Op1) {
17094 if (RetVT.SimpleTy != MVT::f128)
17095 return Register();
17096 if ((Subtarget->hasVLX())) {
17097 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17098 }
17099 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17100 return fastEmitInst_rr(MachineInstOpcode: X86::VXORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17101 }
17102 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17103 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17104 }
17105 return Register();
17106}
17107
17108Register fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17109 if (RetVT.SimpleTy != MVT::v4f32)
17110 return Register();
17111 return fastEmitInst_rr(MachineInstOpcode: X86::XORPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17112}
17113
17114Register fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17115 switch (VT.SimpleTy) {
17116 case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op1);
17117 case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op1);
17118 default: return Register();
17119 }
17120}
17121
17122// FastEmit functions for X86ISD::GF2P8MULB.
17123
17124Register fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17125 if (RetVT.SimpleTy != MVT::v16i8)
17126 return Register();
17127 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
17128 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17129 }
17130 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
17131 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
17132 }
17133 if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17134 return fastEmitInst_rr(MachineInstOpcode: X86::GF2P8MULBrr, RC: &X86::VR128RegClass, Op0, Op1);
17135 }
17136 return Register();
17137}
17138
17139Register fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17140 if (RetVT.SimpleTy != MVT::v32i8)
17141 return Register();
17142 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
17143 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17144 }
17145 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
17146 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17147 }
17148 return Register();
17149}
17150
17151Register fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17152 if (RetVT.SimpleTy != MVT::v64i8)
17153 return Register();
17154 if ((Subtarget->hasAVX512()) && (Subtarget->hasGFNI())) {
17155 return fastEmitInst_rr(MachineInstOpcode: X86::VGF2P8MULBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17156 }
17157 return Register();
17158}
17159
17160Register fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17161 switch (VT.SimpleTy) {
17162 case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op1);
17163 case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op1);
17164 case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op1);
17165 default: return Register();
17166 }
17167}
17168
17169// FastEmit functions for X86ISD::HADD.
17170
17171Register fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17172 if (RetVT.SimpleTy != MVT::v8i16)
17173 return Register();
17174 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17175 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17176 }
17177 if ((Subtarget->hasAVX())) {
17178 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17179 }
17180 return Register();
17181}
17182
17183Register fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17184 if (RetVT.SimpleTy != MVT::v16i16)
17185 return Register();
17186 if ((Subtarget->hasAVX2())) {
17187 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17188 }
17189 return Register();
17190}
17191
17192Register fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17193 if (RetVT.SimpleTy != MVT::v4i32)
17194 return Register();
17195 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17196 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
17197 }
17198 if ((Subtarget->hasAVX())) {
17199 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDrr, RC: &X86::VR128RegClass, Op0, Op1);
17200 }
17201 return Register();
17202}
17203
17204Register fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17205 if (RetVT.SimpleTy != MVT::v8i32)
17206 return Register();
17207 if ((Subtarget->hasAVX2())) {
17208 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17209 }
17210 return Register();
17211}
17212
17213Register fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17214 switch (VT.SimpleTy) {
17215 case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op1);
17216 case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op1);
17217 case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op1);
17218 case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op1);
17219 default: return Register();
17220 }
17221}
17222
17223// FastEmit functions for X86ISD::HADDS.
17224
17225Register fastEmit_X86ISD_HADDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17226 if (RetVT.SimpleTy != MVT::v8i16)
17227 return Register();
17228 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17229 return fastEmitInst_rr(MachineInstOpcode: X86::PHADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17230 }
17231 if ((Subtarget->hasAVX())) {
17232 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17233 }
17234 return Register();
17235}
17236
17237Register fastEmit_X86ISD_HADDS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17238 if (RetVT.SimpleTy != MVT::v16i16)
17239 return Register();
17240 if ((Subtarget->hasAVX2())) {
17241 return fastEmitInst_rr(MachineInstOpcode: X86::VPHADDSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17242 }
17243 return Register();
17244}
17245
17246Register fastEmit_X86ISD_HADDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17247 switch (VT.SimpleTy) {
17248 case MVT::v8i16: return fastEmit_X86ISD_HADDS_MVT_v8i16_rr(RetVT, Op0, Op1);
17249 case MVT::v16i16: return fastEmit_X86ISD_HADDS_MVT_v16i16_rr(RetVT, Op0, Op1);
17250 default: return Register();
17251 }
17252}
17253
17254// FastEmit functions for X86ISD::HSUB.
17255
17256Register fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17257 if (RetVT.SimpleTy != MVT::v8i16)
17258 return Register();
17259 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17260 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17261 }
17262 if ((Subtarget->hasAVX())) {
17263 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWrr, RC: &X86::VR128RegClass, Op0, Op1);
17264 }
17265 return Register();
17266}
17267
17268Register fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17269 if (RetVT.SimpleTy != MVT::v16i16)
17270 return Register();
17271 if ((Subtarget->hasAVX2())) {
17272 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17273 }
17274 return Register();
17275}
17276
17277Register fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17278 if (RetVT.SimpleTy != MVT::v4i32)
17279 return Register();
17280 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17281 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
17282 }
17283 if ((Subtarget->hasAVX())) {
17284 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDrr, RC: &X86::VR128RegClass, Op0, Op1);
17285 }
17286 return Register();
17287}
17288
17289Register fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17290 if (RetVT.SimpleTy != MVT::v8i32)
17291 return Register();
17292 if ((Subtarget->hasAVX2())) {
17293 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17294 }
17295 return Register();
17296}
17297
17298Register fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17299 switch (VT.SimpleTy) {
17300 case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op1);
17301 case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op1);
17302 case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op1);
17303 case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op1);
17304 default: return Register();
17305 }
17306}
17307
17308// FastEmit functions for X86ISD::HSUBS.
17309
17310Register fastEmit_X86ISD_HSUBS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17311 if (RetVT.SimpleTy != MVT::v8i16)
17312 return Register();
17313 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17314 return fastEmitInst_rr(MachineInstOpcode: X86::PHSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17315 }
17316 if ((Subtarget->hasAVX())) {
17317 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17318 }
17319 return Register();
17320}
17321
17322Register fastEmit_X86ISD_HSUBS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17323 if (RetVT.SimpleTy != MVT::v16i16)
17324 return Register();
17325 if ((Subtarget->hasAVX2())) {
17326 return fastEmitInst_rr(MachineInstOpcode: X86::VPHSUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17327 }
17328 return Register();
17329}
17330
17331Register fastEmit_X86ISD_HSUBS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17332 switch (VT.SimpleTy) {
17333 case MVT::v8i16: return fastEmit_X86ISD_HSUBS_MVT_v8i16_rr(RetVT, Op0, Op1);
17334 case MVT::v16i16: return fastEmit_X86ISD_HSUBS_MVT_v16i16_rr(RetVT, Op0, Op1);
17335 default: return Register();
17336 }
17337}
17338
17339// FastEmit functions for X86ISD::KADD.
17340
17341Register fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
17342 if (RetVT.SimpleTy != MVT::v8i1)
17343 return Register();
17344 if ((Subtarget->hasDQI())) {
17345 return fastEmitInst_rr(MachineInstOpcode: X86::KADDBkk, RC: &X86::VK8RegClass, Op0, Op1);
17346 }
17347 return Register();
17348}
17349
17350Register fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
17351 if (RetVT.SimpleTy != MVT::v16i1)
17352 return Register();
17353 if ((Subtarget->hasDQI())) {
17354 return fastEmitInst_rr(MachineInstOpcode: X86::KADDWkk, RC: &X86::VK16RegClass, Op0, Op1);
17355 }
17356 return Register();
17357}
17358
17359Register fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
17360 if (RetVT.SimpleTy != MVT::v32i1)
17361 return Register();
17362 if ((Subtarget->hasBWI())) {
17363 return fastEmitInst_rr(MachineInstOpcode: X86::KADDDkk, RC: &X86::VK32RegClass, Op0, Op1);
17364 }
17365 return Register();
17366}
17367
17368Register fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
17369 if (RetVT.SimpleTy != MVT::v64i1)
17370 return Register();
17371 if ((Subtarget->hasBWI())) {
17372 return fastEmitInst_rr(MachineInstOpcode: X86::KADDQkk, RC: &X86::VK64RegClass, Op0, Op1);
17373 }
17374 return Register();
17375}
17376
17377Register fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17378 switch (VT.SimpleTy) {
17379 case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op1);
17380 case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op1);
17381 case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op1);
17382 case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op1);
17383 default: return Register();
17384 }
17385}
17386
17387// FastEmit functions for X86ISD::KORTEST.
17388
17389Register fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
17390 if (RetVT.SimpleTy != MVT::i32)
17391 return Register();
17392 if ((Subtarget->hasDQI())) {
17393 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTBkk, RC: &X86::VK8RegClass, Op0, Op1);
17394 }
17395 return Register();
17396}
17397
17398Register fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
17399 if (RetVT.SimpleTy != MVT::i32)
17400 return Register();
17401 if ((Subtarget->hasAVX512())) {
17402 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTWkk, RC: &X86::VK16RegClass, Op0, Op1);
17403 }
17404 return Register();
17405}
17406
17407Register fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
17408 if (RetVT.SimpleTy != MVT::i32)
17409 return Register();
17410 if ((Subtarget->hasBWI())) {
17411 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTDkk, RC: &X86::VK32RegClass, Op0, Op1);
17412 }
17413 return Register();
17414}
17415
17416Register fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
17417 if (RetVT.SimpleTy != MVT::i32)
17418 return Register();
17419 if ((Subtarget->hasBWI())) {
17420 return fastEmitInst_rr(MachineInstOpcode: X86::KORTESTQkk, RC: &X86::VK64RegClass, Op0, Op1);
17421 }
17422 return Register();
17423}
17424
17425Register fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17426 switch (VT.SimpleTy) {
17427 case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
17428 case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
17429 case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
17430 case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
17431 default: return Register();
17432 }
17433}
17434
17435// FastEmit functions for X86ISD::KTEST.
17436
17437Register fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, Register Op0, Register Op1) {
17438 if (RetVT.SimpleTy != MVT::i32)
17439 return Register();
17440 if ((Subtarget->hasDQI())) {
17441 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTBkk, RC: &X86::VK8RegClass, Op0, Op1);
17442 }
17443 return Register();
17444}
17445
17446Register fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, Register Op0, Register Op1) {
17447 if (RetVT.SimpleTy != MVT::i32)
17448 return Register();
17449 if ((Subtarget->hasDQI())) {
17450 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTWkk, RC: &X86::VK16RegClass, Op0, Op1);
17451 }
17452 return Register();
17453}
17454
17455Register fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, Register Op0, Register Op1) {
17456 if (RetVT.SimpleTy != MVT::i32)
17457 return Register();
17458 if ((Subtarget->hasBWI())) {
17459 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTDkk, RC: &X86::VK32RegClass, Op0, Op1);
17460 }
17461 return Register();
17462}
17463
17464Register fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, Register Op0, Register Op1) {
17465 if (RetVT.SimpleTy != MVT::i32)
17466 return Register();
17467 if ((Subtarget->hasBWI())) {
17468 return fastEmitInst_rr(MachineInstOpcode: X86::KTESTQkk, RC: &X86::VK64RegClass, Op0, Op1);
17469 }
17470 return Register();
17471}
17472
17473Register fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17474 switch (VT.SimpleTy) {
17475 case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
17476 case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
17477 case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
17478 case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
17479 default: return Register();
17480 }
17481}
17482
17483// FastEmit functions for X86ISD::MOVHLPS.
17484
17485Register fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17486 if (RetVT.SimpleTy != MVT::v4f32)
17487 return Register();
17488 if ((Subtarget->hasAVX512())) {
17489 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17490 }
17491 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17492 return fastEmitInst_rr(MachineInstOpcode: X86::MOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17493 }
17494 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17495 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVHLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17496 }
17497 return Register();
17498}
17499
17500Register fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17501 switch (VT.SimpleTy) {
17502 case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op1);
17503 default: return Register();
17504 }
17505}
17506
17507// FastEmit functions for X86ISD::MOVLHPS.
17508
17509Register fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17510 if (RetVT.SimpleTy != MVT::v4f32)
17511 return Register();
17512 if ((Subtarget->hasAVX512())) {
17513 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17514 }
17515 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17516 return fastEmitInst_rr(MachineInstOpcode: X86::MOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17517 }
17518 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17519 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVLHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
17520 }
17521 return Register();
17522}
17523
17524Register fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17525 switch (VT.SimpleTy) {
17526 case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op1);
17527 default: return Register();
17528 }
17529}
17530
17531// FastEmit functions for X86ISD::MOVSD.
17532
17533Register fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
17534 if (RetVT.SimpleTy != MVT::v2f64)
17535 return Register();
17536 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
17537 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17538 }
17539 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17540 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
17541 }
17542 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17543 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSDrr, RC: &X86::VR128RegClass, Op0, Op1);
17544 }
17545 return Register();
17546}
17547
17548Register fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17549 switch (VT.SimpleTy) {
17550 case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op1);
17551 default: return Register();
17552 }
17553}
17554
17555// FastEmit functions for X86ISD::MOVSH.
17556
17557Register fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
17558 if (RetVT.SimpleTy != MVT::v8f16)
17559 return Register();
17560 if ((Subtarget->hasFP16())) {
17561 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17562 }
17563 return Register();
17564}
17565
17566Register fastEmit_X86ISD_MOVSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17567 switch (VT.SimpleTy) {
17568 case MVT::v8f16: return fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(RetVT, Op0, Op1);
17569 default: return Register();
17570 }
17571}
17572
17573// FastEmit functions for X86ISD::MOVSS.
17574
17575Register fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
17576 if (RetVT.SimpleTy != MVT::v4f32)
17577 return Register();
17578 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
17579 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
17580 }
17581 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
17582 return fastEmitInst_rr(MachineInstOpcode: X86::MOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
17583 }
17584 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
17585 return fastEmitInst_rr(MachineInstOpcode: X86::VMOVSSrr, RC: &X86::VR128RegClass, Op0, Op1);
17586 }
17587 return Register();
17588}
17589
17590Register fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17591 switch (VT.SimpleTy) {
17592 case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op1);
17593 default: return Register();
17594 }
17595}
17596
17597// FastEmit functions for X86ISD::MULHRS.
17598
17599Register fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17600 if (RetVT.SimpleTy != MVT::v8i16)
17601 return Register();
17602 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17603 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17604 }
17605 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17606 return fastEmitInst_rr(MachineInstOpcode: X86::PMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17607 }
17608 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17609 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWrr, RC: &X86::VR128RegClass, Op0, Op1);
17610 }
17611 return Register();
17612}
17613
17614Register fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17615 if (RetVT.SimpleTy != MVT::v16i16)
17616 return Register();
17617 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17618 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17619 }
17620 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17621 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17622 }
17623 return Register();
17624}
17625
17626Register fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17627 if (RetVT.SimpleTy != MVT::v32i16)
17628 return Register();
17629 if ((Subtarget->hasBWI())) {
17630 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULHRSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17631 }
17632 return Register();
17633}
17634
17635Register fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17636 switch (VT.SimpleTy) {
17637 case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op1);
17638 case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op1);
17639 case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op1);
17640 default: return Register();
17641 }
17642}
17643
17644// FastEmit functions for X86ISD::MULTISHIFT.
17645
17646Register fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17647 if (RetVT.SimpleTy != MVT::v16i8)
17648 return Register();
17649 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17650 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17651 }
17652 return Register();
17653}
17654
17655Register fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17656 if (RetVT.SimpleTy != MVT::v32i8)
17657 return Register();
17658 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17659 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17660 }
17661 return Register();
17662}
17663
17664Register fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
17665 if (RetVT.SimpleTy != MVT::v64i8)
17666 return Register();
17667 if ((Subtarget->hasVBMI())) {
17668 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULTISHIFTQBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17669 }
17670 return Register();
17671}
17672
17673Register fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17674 switch (VT.SimpleTy) {
17675 case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op1);
17676 case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op1);
17677 case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op1);
17678 default: return Register();
17679 }
17680}
17681
17682// FastEmit functions for X86ISD::PACKSS.
17683
17684Register fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17685 if (RetVT.SimpleTy != MVT::v16i8)
17686 return Register();
17687 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17688 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17689 }
17690 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17691 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17692 }
17693 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17694 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17695 }
17696 return Register();
17697}
17698
17699Register fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17700 if (RetVT.SimpleTy != MVT::v32i8)
17701 return Register();
17702 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17703 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17704 }
17705 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17706 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17707 }
17708 return Register();
17709}
17710
17711Register fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17712 if (RetVT.SimpleTy != MVT::v64i8)
17713 return Register();
17714 if ((Subtarget->hasBWI())) {
17715 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17716 }
17717 return Register();
17718}
17719
17720Register fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17721 if (RetVT.SimpleTy != MVT::v8i16)
17722 return Register();
17723 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17724 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17725 }
17726 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17727 return fastEmitInst_rr(MachineInstOpcode: X86::PACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17728 }
17729 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17730 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17731 }
17732 return Register();
17733}
17734
17735Register fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17736 if (RetVT.SimpleTy != MVT::v16i16)
17737 return Register();
17738 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17739 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17740 }
17741 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17742 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17743 }
17744 return Register();
17745}
17746
17747Register fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
17748 if (RetVT.SimpleTy != MVT::v32i16)
17749 return Register();
17750 if ((Subtarget->hasBWI())) {
17751 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKSSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17752 }
17753 return Register();
17754}
17755
17756Register fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17757 switch (VT.SimpleTy) {
17758 case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op1);
17759 case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op1);
17760 case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op1);
17761 case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op1);
17762 case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op1);
17763 case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op1);
17764 default: return Register();
17765 }
17766}
17767
17768// FastEmit functions for X86ISD::PACKUS.
17769
17770Register fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17771 if (RetVT.SimpleTy != MVT::v16i8)
17772 return Register();
17773 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17774 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17775 }
17776 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17777 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17778 }
17779 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17780 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBrr, RC: &X86::VR128RegClass, Op0, Op1);
17781 }
17782 return Register();
17783}
17784
17785Register fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17786 if (RetVT.SimpleTy != MVT::v32i8)
17787 return Register();
17788 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17789 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17790 }
17791 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17792 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17793 }
17794 return Register();
17795}
17796
17797Register fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
17798 if (RetVT.SimpleTy != MVT::v64i8)
17799 return Register();
17800 if ((Subtarget->hasBWI())) {
17801 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSWBZrr, RC: &X86::VR512RegClass, Op0, Op1);
17802 }
17803 return Register();
17804}
17805
17806Register fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17807 if (RetVT.SimpleTy != MVT::v8i16)
17808 return Register();
17809 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17810 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
17811 }
17812 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17813 return fastEmitInst_rr(MachineInstOpcode: X86::PACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17814 }
17815 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17816 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWrr, RC: &X86::VR128RegClass, Op0, Op1);
17817 }
17818 return Register();
17819}
17820
17821Register fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17822 if (RetVT.SimpleTy != MVT::v16i16)
17823 return Register();
17824 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17825 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
17826 }
17827 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17828 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17829 }
17830 return Register();
17831}
17832
17833Register fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
17834 if (RetVT.SimpleTy != MVT::v32i16)
17835 return Register();
17836 if ((Subtarget->hasBWI())) {
17837 return fastEmitInst_rr(MachineInstOpcode: X86::VPACKUSDWZrr, RC: &X86::VR512RegClass, Op0, Op1);
17838 }
17839 return Register();
17840}
17841
17842Register fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17843 switch (VT.SimpleTy) {
17844 case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op1);
17845 case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op1);
17846 case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op1);
17847 case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op1);
17848 case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op1);
17849 case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op1);
17850 default: return Register();
17851 }
17852}
17853
17854// FastEmit functions for X86ISD::PCMPEQ.
17855
17856Register fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17857 if (RetVT.SimpleTy != MVT::v16i8)
17858 return Register();
17859 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17860 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
17861 }
17862 if ((Subtarget->hasAVX()) && (true)) {
17863 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBrr, RC: &X86::VR128RegClass, Op0, Op1);
17864 }
17865 return Register();
17866}
17867
17868Register fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17869 if (RetVT.SimpleTy != MVT::v32i8)
17870 return Register();
17871 if ((Subtarget->hasAVX2()) && (true)) {
17872 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17873 }
17874 return Register();
17875}
17876
17877Register fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17878 if (RetVT.SimpleTy != MVT::v8i16)
17879 return Register();
17880 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17881 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
17882 }
17883 if ((Subtarget->hasAVX()) && (true)) {
17884 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWrr, RC: &X86::VR128RegClass, Op0, Op1);
17885 }
17886 return Register();
17887}
17888
17889Register fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17890 if (RetVT.SimpleTy != MVT::v16i16)
17891 return Register();
17892 if ((Subtarget->hasAVX2()) && (true)) {
17893 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17894 }
17895 return Register();
17896}
17897
17898Register fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17899 if (RetVT.SimpleTy != MVT::v4i32)
17900 return Register();
17901 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17902 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
17903 }
17904 if ((Subtarget->hasAVX()) && (true)) {
17905 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDrr, RC: &X86::VR128RegClass, Op0, Op1);
17906 }
17907 return Register();
17908}
17909
17910Register fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
17911 if (RetVT.SimpleTy != MVT::v8i32)
17912 return Register();
17913 if ((Subtarget->hasAVX2()) && (true)) {
17914 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQDYrr, RC: &X86::VR256RegClass, Op0, Op1);
17915 }
17916 return Register();
17917}
17918
17919Register fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
17920 if (RetVT.SimpleTy != MVT::v2i64)
17921 return Register();
17922 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
17923 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
17924 }
17925 if ((Subtarget->hasAVX())) {
17926 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQrr, RC: &X86::VR128RegClass, Op0, Op1);
17927 }
17928 return Register();
17929}
17930
17931Register fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
17932 if (RetVT.SimpleTy != MVT::v4i64)
17933 return Register();
17934 if ((Subtarget->hasAVX2())) {
17935 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPEQQYrr, RC: &X86::VR256RegClass, Op0, Op1);
17936 }
17937 return Register();
17938}
17939
17940Register fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
17941 switch (VT.SimpleTy) {
17942 case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op1);
17943 case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op1);
17944 case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op1);
17945 case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op1);
17946 case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op1);
17947 case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op1);
17948 case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op1);
17949 case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op1);
17950 default: return Register();
17951 }
17952}
17953
17954// FastEmit functions for X86ISD::PCMPGT.
17955
17956Register fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
17957 if (RetVT.SimpleTy != MVT::v16i8)
17958 return Register();
17959 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17960 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
17961 }
17962 if ((Subtarget->hasAVX()) && (true)) {
17963 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBrr, RC: &X86::VR128RegClass, Op0, Op1);
17964 }
17965 return Register();
17966}
17967
17968Register fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
17969 if (RetVT.SimpleTy != MVT::v32i8)
17970 return Register();
17971 if ((Subtarget->hasAVX2()) && (true)) {
17972 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTBYrr, RC: &X86::VR256RegClass, Op0, Op1);
17973 }
17974 return Register();
17975}
17976
17977Register fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
17978 if (RetVT.SimpleTy != MVT::v8i16)
17979 return Register();
17980 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17981 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
17982 }
17983 if ((Subtarget->hasAVX()) && (true)) {
17984 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWrr, RC: &X86::VR128RegClass, Op0, Op1);
17985 }
17986 return Register();
17987}
17988
17989Register fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
17990 if (RetVT.SimpleTy != MVT::v16i16)
17991 return Register();
17992 if ((Subtarget->hasAVX2()) && (true)) {
17993 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTWYrr, RC: &X86::VR256RegClass, Op0, Op1);
17994 }
17995 return Register();
17996}
17997
17998Register fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
17999 if (RetVT.SimpleTy != MVT::v4i32)
18000 return Register();
18001 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18002 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
18003 }
18004 if ((Subtarget->hasAVX()) && (true)) {
18005 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDrr, RC: &X86::VR128RegClass, Op0, Op1);
18006 }
18007 return Register();
18008}
18009
18010Register fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
18011 if (RetVT.SimpleTy != MVT::v8i32)
18012 return Register();
18013 if ((Subtarget->hasAVX2()) && (true)) {
18014 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18015 }
18016 return Register();
18017}
18018
18019Register fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
18020 if (RetVT.SimpleTy != MVT::v2i64)
18021 return Register();
18022 if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) {
18023 return fastEmitInst_rr(MachineInstOpcode: X86::PCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
18024 }
18025 if ((Subtarget->hasAVX())) {
18026 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQrr, RC: &X86::VR128RegClass, Op0, Op1);
18027 }
18028 return Register();
18029}
18030
18031Register fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
18032 if (RetVT.SimpleTy != MVT::v4i64)
18033 return Register();
18034 if ((Subtarget->hasAVX2())) {
18035 return fastEmitInst_rr(MachineInstOpcode: X86::VPCMPGTQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18036 }
18037 return Register();
18038}
18039
18040Register fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18041 switch (VT.SimpleTy) {
18042 case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op1);
18043 case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op1);
18044 case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op1);
18045 case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op1);
18046 case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op1);
18047 case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op1);
18048 case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op1);
18049 case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op1);
18050 default: return Register();
18051 }
18052}
18053
18054// FastEmit functions for X86ISD::PMULDQ.
18055
18056Register fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
18057 if (RetVT.SimpleTy != MVT::v2i64)
18058 return Register();
18059 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18060 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18061 }
18062 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
18063 return fastEmitInst_rr(MachineInstOpcode: X86::PMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18064 }
18065 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18066 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18067 }
18068 return Register();
18069}
18070
18071Register fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
18072 if (RetVT.SimpleTy != MVT::v4i64)
18073 return Register();
18074 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18075 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18076 }
18077 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18078 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18079 }
18080 return Register();
18081}
18082
18083Register fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
18084 if (RetVT.SimpleTy != MVT::v8i64)
18085 return Register();
18086 if ((Subtarget->hasAVX512())) {
18087 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18088 }
18089 return Register();
18090}
18091
18092Register fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18093 switch (VT.SimpleTy) {
18094 case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
18095 case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
18096 case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
18097 default: return Register();
18098 }
18099}
18100
18101// FastEmit functions for X86ISD::PMULUDQ.
18102
18103Register fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
18104 if (RetVT.SimpleTy != MVT::v2i64)
18105 return Register();
18106 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18107 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18108 }
18109 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18110 return fastEmitInst_rr(MachineInstOpcode: X86::PMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18111 }
18112 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18113 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18114 }
18115 return Register();
18116}
18117
18118Register fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
18119 if (RetVT.SimpleTy != MVT::v4i64)
18120 return Register();
18121 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18122 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18123 }
18124 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18125 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18126 }
18127 return Register();
18128}
18129
18130Register fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
18131 if (RetVT.SimpleTy != MVT::v8i64)
18132 return Register();
18133 if ((Subtarget->hasAVX512())) {
18134 return fastEmitInst_rr(MachineInstOpcode: X86::VPMULUDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18135 }
18136 return Register();
18137}
18138
18139Register fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18140 switch (VT.SimpleTy) {
18141 case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
18142 case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
18143 case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
18144 default: return Register();
18145 }
18146}
18147
18148// FastEmit functions for X86ISD::PSADBW.
18149
18150Register fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18151 if (RetVT.SimpleTy != MVT::v2i64)
18152 return Register();
18153 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18154 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18155 }
18156 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18157 return fastEmitInst_rr(MachineInstOpcode: X86::PSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18158 }
18159 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18160 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18161 }
18162 return Register();
18163}
18164
18165Register fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18166 if (RetVT.SimpleTy != MVT::v4i64)
18167 return Register();
18168 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18169 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18170 }
18171 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18172 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
18173 }
18174 return Register();
18175}
18176
18177Register fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
18178 if (RetVT.SimpleTy != MVT::v8i64)
18179 return Register();
18180 if ((Subtarget->hasBWI())) {
18181 return fastEmitInst_rr(MachineInstOpcode: X86::VPSADBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
18182 }
18183 return Register();
18184}
18185
18186Register fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18187 switch (VT.SimpleTy) {
18188 case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op1);
18189 case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op1);
18190 case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op1);
18191 default: return Register();
18192 }
18193}
18194
18195// FastEmit functions for X86ISD::PSHUFB.
18196
18197Register fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18198 if (RetVT.SimpleTy != MVT::v16i8)
18199 return Register();
18200 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18201 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18202 }
18203 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
18204 return fastEmitInst_rr(MachineInstOpcode: X86::PSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
18205 }
18206 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18207 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBrr, RC: &X86::VR128RegClass, Op0, Op1);
18208 }
18209 return Register();
18210}
18211
18212Register fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18213 if (RetVT.SimpleTy != MVT::v32i8)
18214 return Register();
18215 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18216 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18217 }
18218 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18219 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBYrr, RC: &X86::VR256RegClass, Op0, Op1);
18220 }
18221 return Register();
18222}
18223
18224Register fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
18225 if (RetVT.SimpleTy != MVT::v64i8)
18226 return Register();
18227 if ((Subtarget->hasBWI())) {
18228 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBZrr, RC: &X86::VR512RegClass, Op0, Op1);
18229 }
18230 return Register();
18231}
18232
18233Register fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18234 switch (VT.SimpleTy) {
18235 case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op1);
18236 case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op1);
18237 case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op1);
18238 default: return Register();
18239 }
18240}
18241
18242// FastEmit functions for X86ISD::PTEST.
18243
18244Register fastEmit_X86ISD_PTEST_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
18245 if (RetVT.SimpleTy != MVT::i32)
18246 return Register();
18247 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
18248 return fastEmitInst_rr(MachineInstOpcode: X86::PTESTrr, RC: &X86::VR128RegClass, Op0, Op1);
18249 }
18250 if ((Subtarget->hasAVX())) {
18251 return fastEmitInst_rr(MachineInstOpcode: X86::VPTESTrr, RC: &X86::VR128RegClass, Op0, Op1);
18252 }
18253 return Register();
18254}
18255
18256Register fastEmit_X86ISD_PTEST_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
18257 if (RetVT.SimpleTy != MVT::i32)
18258 return Register();
18259 if ((Subtarget->hasAVX())) {
18260 return fastEmitInst_rr(MachineInstOpcode: X86::VPTESTYrr, RC: &X86::VR256RegClass, Op0, Op1);
18261 }
18262 return Register();
18263}
18264
18265Register fastEmit_X86ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18266 switch (VT.SimpleTy) {
18267 case MVT::v2i64: return fastEmit_X86ISD_PTEST_MVT_v2i64_rr(RetVT, Op0, Op1);
18268 case MVT::v4i64: return fastEmit_X86ISD_PTEST_MVT_v4i64_rr(RetVT, Op0, Op1);
18269 default: return Register();
18270 }
18271}
18272
18273// FastEmit functions for X86ISD::RCP14S.
18274
18275Register fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18276 if (RetVT.SimpleTy != MVT::v8f16)
18277 return Register();
18278 if ((Subtarget->hasFP16())) {
18279 return fastEmitInst_rr(MachineInstOpcode: X86::VRCPSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18280 }
18281 return Register();
18282}
18283
18284Register fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18285 if (RetVT.SimpleTy != MVT::v4f32)
18286 return Register();
18287 if ((Subtarget->hasAVX512())) {
18288 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18289 }
18290 return Register();
18291}
18292
18293Register fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18294 if (RetVT.SimpleTy != MVT::v2f64)
18295 return Register();
18296 if ((Subtarget->hasAVX512())) {
18297 return fastEmitInst_rr(MachineInstOpcode: X86::VRCP14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18298 }
18299 return Register();
18300}
18301
18302Register fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18303 switch (VT.SimpleTy) {
18304 case MVT::v8f16: return fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(RetVT, Op0, Op1);
18305 case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op1);
18306 case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op1);
18307 default: return Register();
18308 }
18309}
18310
18311// FastEmit functions for X86ISD::RSQRT14S.
18312
18313Register fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18314 if (RetVT.SimpleTy != MVT::v8f16)
18315 return Register();
18316 if ((Subtarget->hasFP16())) {
18317 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRTSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18318 }
18319 return Register();
18320}
18321
18322Register fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18323 if (RetVT.SimpleTy != MVT::v4f32)
18324 return Register();
18325 if ((Subtarget->hasAVX512())) {
18326 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18327 }
18328 return Register();
18329}
18330
18331Register fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18332 if (RetVT.SimpleTy != MVT::v2f64)
18333 return Register();
18334 if ((Subtarget->hasAVX512())) {
18335 return fastEmitInst_rr(MachineInstOpcode: X86::VRSQRT14SDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18336 }
18337 return Register();
18338}
18339
18340Register fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18341 switch (VT.SimpleTy) {
18342 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(RetVT, Op0, Op1);
18343 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op1);
18344 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op1);
18345 default: return Register();
18346 }
18347}
18348
18349// FastEmit functions for X86ISD::SCALEF.
18350
18351Register fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18352 if (RetVT.SimpleTy != MVT::v8f16)
18353 return Register();
18354 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18355 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18356 }
18357 return Register();
18358}
18359
18360Register fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
18361 if (RetVT.SimpleTy != MVT::v16f16)
18362 return Register();
18363 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
18364 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18365 }
18366 return Register();
18367}
18368
18369Register fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
18370 if (RetVT.SimpleTy != MVT::v32f16)
18371 return Register();
18372 if ((Subtarget->hasFP16())) {
18373 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
18374 }
18375 return Register();
18376}
18377
18378Register fastEmit_X86ISD_SCALEF_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
18379 if (RetVT.SimpleTy != MVT::v8bf16)
18380 return Register();
18381 if ((Subtarget->hasAVX10_2())) {
18382 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18383 }
18384 return Register();
18385}
18386
18387Register fastEmit_X86ISD_SCALEF_MVT_v16bf16_rr(MVT RetVT, Register Op0, Register Op1) {
18388 if (RetVT.SimpleTy != MVT::v16bf16)
18389 return Register();
18390 if ((Subtarget->hasAVX10_2())) {
18391 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18392 }
18393 return Register();
18394}
18395
18396Register fastEmit_X86ISD_SCALEF_MVT_v32bf16_rr(MVT RetVT, Register Op0, Register Op1) {
18397 if (RetVT.SimpleTy != MVT::v32bf16)
18398 return Register();
18399 if ((Subtarget->hasAVX10_2())) {
18400 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFBF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
18401 }
18402 return Register();
18403}
18404
18405Register fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18406 if (RetVT.SimpleTy != MVT::v4f32)
18407 return Register();
18408 if ((Subtarget->hasVLX())) {
18409 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18410 }
18411 return Register();
18412}
18413
18414Register fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18415 if (RetVT.SimpleTy != MVT::v8f32)
18416 return Register();
18417 if ((Subtarget->hasVLX())) {
18418 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18419 }
18420 return Register();
18421}
18422
18423Register fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
18424 if (RetVT.SimpleTy != MVT::v16f32)
18425 return Register();
18426 if ((Subtarget->hasAVX512())) {
18427 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
18428 }
18429 return Register();
18430}
18431
18432Register fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18433 if (RetVT.SimpleTy != MVT::v2f64)
18434 return Register();
18435 if ((Subtarget->hasVLX())) {
18436 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18437 }
18438 return Register();
18439}
18440
18441Register fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18442 if (RetVT.SimpleTy != MVT::v4f64)
18443 return Register();
18444 if ((Subtarget->hasVLX())) {
18445 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18446 }
18447 return Register();
18448}
18449
18450Register fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
18451 if (RetVT.SimpleTy != MVT::v8f64)
18452 return Register();
18453 if ((Subtarget->hasAVX512())) {
18454 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18455 }
18456 return Register();
18457}
18458
18459Register fastEmit_X86ISD_SCALEF_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18460 switch (VT.SimpleTy) {
18461 case MVT::v8f16: return fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(RetVT, Op0, Op1);
18462 case MVT::v16f16: return fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(RetVT, Op0, Op1);
18463 case MVT::v32f16: return fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(RetVT, Op0, Op1);
18464 case MVT::v8bf16: return fastEmit_X86ISD_SCALEF_MVT_v8bf16_rr(RetVT, Op0, Op1);
18465 case MVT::v16bf16: return fastEmit_X86ISD_SCALEF_MVT_v16bf16_rr(RetVT, Op0, Op1);
18466 case MVT::v32bf16: return fastEmit_X86ISD_SCALEF_MVT_v32bf16_rr(RetVT, Op0, Op1);
18467 case MVT::v4f32: return fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(RetVT, Op0, Op1);
18468 case MVT::v8f32: return fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(RetVT, Op0, Op1);
18469 case MVT::v16f32: return fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(RetVT, Op0, Op1);
18470 case MVT::v2f64: return fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(RetVT, Op0, Op1);
18471 case MVT::v4f64: return fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(RetVT, Op0, Op1);
18472 case MVT::v8f64: return fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(RetVT, Op0, Op1);
18473 default: return Register();
18474 }
18475}
18476
18477// FastEmit functions for X86ISD::SCALEFS.
18478
18479Register fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18480 if (RetVT.SimpleTy != MVT::v8f16)
18481 return Register();
18482 if ((Subtarget->hasFP16())) {
18483 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18484 }
18485 return Register();
18486}
18487
18488Register fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18489 if (RetVT.SimpleTy != MVT::v4f32)
18490 return Register();
18491 if ((Subtarget->hasAVX512())) {
18492 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSSZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18493 }
18494 return Register();
18495}
18496
18497Register fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18498 if (RetVT.SimpleTy != MVT::v2f64)
18499 return Register();
18500 if ((Subtarget->hasAVX512())) {
18501 return fastEmitInst_rr(MachineInstOpcode: X86::VSCALEFSDZrr, RC: &X86::VR128XRegClass, Op0, Op1);
18502 }
18503 return Register();
18504}
18505
18506Register fastEmit_X86ISD_SCALEFS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18507 switch (VT.SimpleTy) {
18508 case MVT::v8f16: return fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(RetVT, Op0, Op1);
18509 case MVT::v4f32: return fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(RetVT, Op0, Op1);
18510 case MVT::v2f64: return fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(RetVT, Op0, Op1);
18511 default: return Register();
18512 }
18513}
18514
18515// FastEmit functions for X86ISD::STRICT_FCMP.
18516
18517Register fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18518 if (RetVT.SimpleTy != MVT::i32)
18519 return Register();
18520 if ((Subtarget->hasFP16())) {
18521 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18522 }
18523 return Register();
18524}
18525
18526Register fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18527 if (RetVT.SimpleTy != MVT::i32)
18528 return Register();
18529 if ((Subtarget->hasAVX512())) {
18530 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18531 }
18532 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18533 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18534 }
18535 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18536 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18537 }
18538 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
18539 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
18540 }
18541 return Register();
18542}
18543
18544Register fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18545 if (RetVT.SimpleTy != MVT::i32)
18546 return Register();
18547 if ((Subtarget->hasAVX512())) {
18548 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18549 }
18550 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18551 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18552 }
18553 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18554 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18555 }
18556 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
18557 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
18558 }
18559 return Register();
18560}
18561
18562Register fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18563 if (RetVT.SimpleTy != MVT::i32)
18564 return Register();
18565 if ((Subtarget->canUseCMOV())) {
18566 return fastEmitInst_rr(MachineInstOpcode: X86::UCOM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
18567 }
18568 return Register();
18569}
18570
18571Register fastEmit_X86ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18572 switch (VT.SimpleTy) {
18573 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
18574 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
18575 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
18576 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
18577 default: return Register();
18578 }
18579}
18580
18581// FastEmit functions for X86ISD::STRICT_FCMPS.
18582
18583Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18584 if (RetVT.SimpleTy != MVT::i32)
18585 return Register();
18586 if ((Subtarget->hasFP16())) {
18587 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18588 }
18589 return Register();
18590}
18591
18592Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18593 if (RetVT.SimpleTy != MVT::i32)
18594 return Register();
18595 if ((Subtarget->hasAVX512())) {
18596 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18597 }
18598 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18599 return fastEmitInst_rr(MachineInstOpcode: X86::COMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18600 }
18601 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18602 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISSrr, RC: &X86::FR32RegClass, Op0, Op1);
18603 }
18604 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
18605 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr32, RC: &X86::RFP32RegClass, Op0, Op1);
18606 }
18607 return Register();
18608}
18609
18610Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18611 if (RetVT.SimpleTy != MVT::i32)
18612 return Register();
18613 if ((Subtarget->hasAVX512())) {
18614 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18615 }
18616 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18617 return fastEmitInst_rr(MachineInstOpcode: X86::COMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18618 }
18619 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18620 return fastEmitInst_rr(MachineInstOpcode: X86::VCOMISDrr, RC: &X86::FR64RegClass, Op0, Op1);
18621 }
18622 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
18623 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr64, RC: &X86::RFP64RegClass, Op0, Op1);
18624 }
18625 return Register();
18626}
18627
18628Register fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18629 if (RetVT.SimpleTy != MVT::i32)
18630 return Register();
18631 if ((Subtarget->canUseCMOV())) {
18632 return fastEmitInst_rr(MachineInstOpcode: X86::COM_FpIr80, RC: &X86::RFP80RegClass, Op0, Op1);
18633 }
18634 return Register();
18635}
18636
18637Register fastEmit_X86ISD_STRICT_FCMPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18638 switch (VT.SimpleTy) {
18639 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(RetVT, Op0, Op1);
18640 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(RetVT, Op0, Op1);
18641 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(RetVT, Op0, Op1);
18642 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(RetVT, Op0, Op1);
18643 default: return Register();
18644 }
18645}
18646
18647// FastEmit functions for X86ISD::STRICT_FP80_ADD.
18648
18649Register fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(MVT RetVT, Register Op0, Register Op1) {
18650 if (RetVT.SimpleTy != MVT::f80)
18651 return Register();
18652 return fastEmitInst_rr(MachineInstOpcode: X86::FP80_ADDr, RC: &X86::RFP80RegClass, Op0, Op1);
18653}
18654
18655Register fastEmit_X86ISD_STRICT_FP80_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18656 switch (VT.SimpleTy) {
18657 case MVT::f80: return fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
18658 default: return Register();
18659 }
18660}
18661
18662// FastEmit functions for X86ISD::TESTP.
18663
18664Register fastEmit_X86ISD_TESTP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18665 if (RetVT.SimpleTy != MVT::i32)
18666 return Register();
18667 if ((Subtarget->hasAVX())) {
18668 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18669 }
18670 return Register();
18671}
18672
18673Register fastEmit_X86ISD_TESTP_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18674 if (RetVT.SimpleTy != MVT::i32)
18675 return Register();
18676 if ((Subtarget->hasAVX())) {
18677 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18678 }
18679 return Register();
18680}
18681
18682Register fastEmit_X86ISD_TESTP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18683 if (RetVT.SimpleTy != MVT::i32)
18684 return Register();
18685 if ((Subtarget->hasAVX())) {
18686 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPDrr, RC: &X86::VR128RegClass, Op0, Op1);
18687 }
18688 return Register();
18689}
18690
18691Register fastEmit_X86ISD_TESTP_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
18692 if (RetVT.SimpleTy != MVT::i32)
18693 return Register();
18694 if ((Subtarget->hasAVX())) {
18695 return fastEmitInst_rr(MachineInstOpcode: X86::VTESTPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18696 }
18697 return Register();
18698}
18699
18700Register fastEmit_X86ISD_TESTP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18701 switch (VT.SimpleTy) {
18702 case MVT::v4f32: return fastEmit_X86ISD_TESTP_MVT_v4f32_rr(RetVT, Op0, Op1);
18703 case MVT::v8f32: return fastEmit_X86ISD_TESTP_MVT_v8f32_rr(RetVT, Op0, Op1);
18704 case MVT::v2f64: return fastEmit_X86ISD_TESTP_MVT_v2f64_rr(RetVT, Op0, Op1);
18705 case MVT::v4f64: return fastEmit_X86ISD_TESTP_MVT_v4f64_rr(RetVT, Op0, Op1);
18706 default: return Register();
18707 }
18708}
18709
18710// FastEmit functions for X86ISD::UCOMI.
18711
18712Register fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18713 if (RetVT.SimpleTy != MVT::i32)
18714 return Register();
18715 if ((Subtarget->hasFP16())) {
18716 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18717 }
18718 return Register();
18719}
18720
18721Register fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18722 if (RetVT.SimpleTy != MVT::i32)
18723 return Register();
18724 if ((Subtarget->hasAVX512())) {
18725 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18726 }
18727 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18728 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18729 }
18730 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18731 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISSrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18732 }
18733 return Register();
18734}
18735
18736Register fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18737 if (RetVT.SimpleTy != MVT::i32)
18738 return Register();
18739 if ((Subtarget->hasAVX512())) {
18740 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18741 }
18742 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18743 return fastEmitInst_rr(MachineInstOpcode: X86::UCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18744 }
18745 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
18746 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMISDrr_Int, RC: &X86::VR128RegClass, Op0, Op1);
18747 }
18748 return Register();
18749}
18750
18751Register fastEmit_X86ISD_UCOMI_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18752 switch (VT.SimpleTy) {
18753 case MVT::v8f16: return fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(RetVT, Op0, Op1);
18754 case MVT::v4f32: return fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(RetVT, Op0, Op1);
18755 case MVT::v2f64: return fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(RetVT, Op0, Op1);
18756 default: return Register();
18757 }
18758}
18759
18760// FastEmit functions for X86ISD::UCOMX.
18761
18762Register fastEmit_X86ISD_UCOMX_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
18763 if (RetVT.SimpleTy != MVT::i32)
18764 return Register();
18765 if ((Subtarget->hasAVX10_2())) {
18766 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSHZrr, RC: &X86::FR16XRegClass, Op0, Op1);
18767 }
18768 return Register();
18769}
18770
18771Register fastEmit_X86ISD_UCOMX_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
18772 if (RetVT.SimpleTy != MVT::i32)
18773 return Register();
18774 if ((Subtarget->hasAVX10_2())) {
18775 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSSZrr, RC: &X86::FR32XRegClass, Op0, Op1);
18776 }
18777 return Register();
18778}
18779
18780Register fastEmit_X86ISD_UCOMX_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
18781 if (RetVT.SimpleTy != MVT::i32)
18782 return Register();
18783 if ((Subtarget->hasAVX10_2())) {
18784 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSDZrr, RC: &X86::FR64XRegClass, Op0, Op1);
18785 }
18786 return Register();
18787}
18788
18789Register fastEmit_X86ISD_UCOMX_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
18790 if (RetVT.SimpleTy != MVT::i32)
18791 return Register();
18792 if ((Subtarget->hasAVX10_2())) {
18793 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSHZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18794 }
18795 return Register();
18796}
18797
18798Register fastEmit_X86ISD_UCOMX_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18799 if (RetVT.SimpleTy != MVT::i32)
18800 return Register();
18801 if ((Subtarget->hasAVX10_2())) {
18802 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSSZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18803 }
18804 return Register();
18805}
18806
18807Register fastEmit_X86ISD_UCOMX_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
18808 if (RetVT.SimpleTy != MVT::i32)
18809 return Register();
18810 if ((Subtarget->hasAVX10_2())) {
18811 return fastEmitInst_rr(MachineInstOpcode: X86::VUCOMXSDZrr_Int, RC: &X86::VR128XRegClass, Op0, Op1);
18812 }
18813 return Register();
18814}
18815
18816Register fastEmit_X86ISD_UCOMX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
18817 switch (VT.SimpleTy) {
18818 case MVT::f16: return fastEmit_X86ISD_UCOMX_MVT_f16_rr(RetVT, Op0, Op1);
18819 case MVT::f32: return fastEmit_X86ISD_UCOMX_MVT_f32_rr(RetVT, Op0, Op1);
18820 case MVT::f64: return fastEmit_X86ISD_UCOMX_MVT_f64_rr(RetVT, Op0, Op1);
18821 case MVT::v8f16: return fastEmit_X86ISD_UCOMX_MVT_v8f16_rr(RetVT, Op0, Op1);
18822 case MVT::v4f32: return fastEmit_X86ISD_UCOMX_MVT_v4f32_rr(RetVT, Op0, Op1);
18823 case MVT::v2f64: return fastEmit_X86ISD_UCOMX_MVT_v2f64_rr(RetVT, Op0, Op1);
18824 default: return Register();
18825 }
18826}
18827
18828// FastEmit functions for X86ISD::UNPCKH.
18829
18830Register fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
18831 if (RetVT.SimpleTy != MVT::v16i8)
18832 return Register();
18833 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18834 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18835 }
18836 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18837 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18838 }
18839 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18840 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWrr, RC: &X86::VR128RegClass, Op0, Op1);
18841 }
18842 return Register();
18843}
18844
18845Register fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
18846 if (RetVT.SimpleTy != MVT::v32i8)
18847 return Register();
18848 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18849 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18850 }
18851 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18852 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
18853 }
18854 return Register();
18855}
18856
18857Register fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
18858 if (RetVT.SimpleTy != MVT::v64i8)
18859 return Register();
18860 if ((Subtarget->hasBWI())) {
18861 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
18862 }
18863 return Register();
18864}
18865
18866Register fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
18867 if (RetVT.SimpleTy != MVT::v8i16)
18868 return Register();
18869 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18870 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18871 }
18872 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18873 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18874 }
18875 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18876 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDrr, RC: &X86::VR128RegClass, Op0, Op1);
18877 }
18878 return Register();
18879}
18880
18881Register fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
18882 if (RetVT.SimpleTy != MVT::v16i16)
18883 return Register();
18884 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
18885 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18886 }
18887 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
18888 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18889 }
18890 return Register();
18891}
18892
18893Register fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
18894 if (RetVT.SimpleTy != MVT::v32i16)
18895 return Register();
18896 if ((Subtarget->hasBWI())) {
18897 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
18898 }
18899 return Register();
18900}
18901
18902Register fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
18903 if (RetVT.SimpleTy != MVT::v4i32)
18904 return Register();
18905 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18906 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18907 }
18908 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18909 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18910 }
18911 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18912 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18913 }
18914 return Register();
18915}
18916
18917Register fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
18918 if (RetVT.SimpleTy != MVT::v8i32)
18919 return Register();
18920 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18921 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
18922 }
18923 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18924 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18925 }
18926 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18927 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18928 }
18929 return Register();
18930}
18931
18932Register fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
18933 if (RetVT.SimpleTy != MVT::v16i32)
18934 return Register();
18935 if ((Subtarget->hasAVX512())) {
18936 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18937 }
18938 return Register();
18939}
18940
18941Register fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
18942 if (RetVT.SimpleTy != MVT::v2i64)
18943 return Register();
18944 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18945 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18946 }
18947 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
18948 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18949 }
18950 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18951 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
18952 }
18953 return Register();
18954}
18955
18956Register fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
18957 if (RetVT.SimpleTy != MVT::v4i64)
18958 return Register();
18959 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
18960 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
18961 }
18962 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18963 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
18964 }
18965 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18966 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
18967 }
18968 return Register();
18969}
18970
18971Register fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
18972 if (RetVT.SimpleTy != MVT::v8i64)
18973 return Register();
18974 if ((Subtarget->hasAVX512())) {
18975 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKHQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
18976 }
18977 return Register();
18978}
18979
18980Register fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
18981 if (RetVT.SimpleTy != MVT::v4f32)
18982 return Register();
18983 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18984 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
18985 }
18986 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
18987 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18988 }
18989 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
18990 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSrr, RC: &X86::VR128RegClass, Op0, Op1);
18991 }
18992 return Register();
18993}
18994
18995Register fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
18996 if (RetVT.SimpleTy != MVT::v8f32)
18997 return Register();
18998 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18999 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19000 }
19001 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19002 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
19003 }
19004 return Register();
19005}
19006
19007Register fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19008 if (RetVT.SimpleTy != MVT::v16f32)
19009 return Register();
19010 if ((Subtarget->hasAVX512())) {
19011 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
19012 }
19013 return Register();
19014}
19015
19016Register fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
19017 if (RetVT.SimpleTy != MVT::v2f64)
19018 return Register();
19019 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19020 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19021 }
19022 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19023 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
19024 }
19025 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19026 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDrr, RC: &X86::VR128RegClass, Op0, Op1);
19027 }
19028 return Register();
19029}
19030
19031Register fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
19032 if (RetVT.SimpleTy != MVT::v4f64)
19033 return Register();
19034 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19035 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19036 }
19037 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19038 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19039 }
19040 return Register();
19041}
19042
19043Register fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
19044 if (RetVT.SimpleTy != MVT::v8f64)
19045 return Register();
19046 if ((Subtarget->hasAVX512())) {
19047 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKHPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19048 }
19049 return Register();
19050}
19051
19052Register fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19053 switch (VT.SimpleTy) {
19054 case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op1);
19055 case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op1);
19056 case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op1);
19057 case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op1);
19058 case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op1);
19059 case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op1);
19060 case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op1);
19061 case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op1);
19062 case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op1);
19063 case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op1);
19064 case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op1);
19065 case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op1);
19066 case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op1);
19067 case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op1);
19068 case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op1);
19069 case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op1);
19070 case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op1);
19071 case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op1);
19072 default: return Register();
19073 }
19074}
19075
19076// FastEmit functions for X86ISD::UNPCKL.
19077
19078Register fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19079 if (RetVT.SimpleTy != MVT::v16i8)
19080 return Register();
19081 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19082 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19083 }
19084 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19085 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
19086 }
19087 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19088 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWrr, RC: &X86::VR128RegClass, Op0, Op1);
19089 }
19090 return Register();
19091}
19092
19093Register fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19094 if (RetVT.SimpleTy != MVT::v32i8)
19095 return Register();
19096 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19097 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19098 }
19099 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19100 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWYrr, RC: &X86::VR256RegClass, Op0, Op1);
19101 }
19102 return Register();
19103}
19104
19105Register fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19106 if (RetVT.SimpleTy != MVT::v64i8)
19107 return Register();
19108 if ((Subtarget->hasBWI())) {
19109 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLBWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19110 }
19111 return Register();
19112}
19113
19114Register fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19115 if (RetVT.SimpleTy != MVT::v8i16)
19116 return Register();
19117 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19118 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19119 }
19120 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19121 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19122 }
19123 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19124 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19125 }
19126 return Register();
19127}
19128
19129Register fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19130 if (RetVT.SimpleTy != MVT::v16i16)
19131 return Register();
19132 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19133 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19134 }
19135 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19136 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19137 }
19138 return Register();
19139}
19140
19141Register fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19142 if (RetVT.SimpleTy != MVT::v32i16)
19143 return Register();
19144 if ((Subtarget->hasBWI())) {
19145 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19146 }
19147 return Register();
19148}
19149
19150Register fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19151 if (RetVT.SimpleTy != MVT::v4i32)
19152 return Register();
19153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19154 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19155 }
19156 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19157 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19158 }
19159 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19160 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19161 }
19162 return Register();
19163}
19164
19165Register fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19166 if (RetVT.SimpleTy != MVT::v8i32)
19167 return Register();
19168 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
19169 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
19170 }
19171 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19172 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19173 }
19174 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19175 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
19176 }
19177 return Register();
19178}
19179
19180Register fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19181 if (RetVT.SimpleTy != MVT::v16i32)
19182 return Register();
19183 if ((Subtarget->hasAVX512())) {
19184 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19185 }
19186 return Register();
19187}
19188
19189Register fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19190 if (RetVT.SimpleTy != MVT::v2i64)
19191 return Register();
19192 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19193 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19194 }
19195 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19196 return fastEmitInst_rr(MachineInstOpcode: X86::PUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19197 }
19198 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19199 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQrr, RC: &X86::VR128RegClass, Op0, Op1);
19200 }
19201 return Register();
19202}
19203
19204Register fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19205 if (RetVT.SimpleTy != MVT::v4i64)
19206 return Register();
19207 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
19208 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19209 }
19210 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19211 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19212 }
19213 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19214 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQYrr, RC: &X86::VR256RegClass, Op0, Op1);
19215 }
19216 return Register();
19217}
19218
19219Register fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19220 if (RetVT.SimpleTy != MVT::v8i64)
19221 return Register();
19222 if ((Subtarget->hasAVX512())) {
19223 return fastEmitInst_rr(MachineInstOpcode: X86::VPUNPCKLQDQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19224 }
19225 return Register();
19226}
19227
19228Register fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19229 if (RetVT.SimpleTy != MVT::v4f32)
19230 return Register();
19231 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19232 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19233 }
19234 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
19235 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
19236 }
19237 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19238 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSrr, RC: &X86::VR128RegClass, Op0, Op1);
19239 }
19240 return Register();
19241}
19242
19243Register fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19244 if (RetVT.SimpleTy != MVT::v8f32)
19245 return Register();
19246 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19247 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19248 }
19249 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19250 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSYrr, RC: &X86::VR256RegClass, Op0, Op1);
19251 }
19252 return Register();
19253}
19254
19255Register fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19256 if (RetVT.SimpleTy != MVT::v16f32)
19257 return Register();
19258 if ((Subtarget->hasAVX512())) {
19259 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPSZrr, RC: &X86::VR512RegClass, Op0, Op1);
19260 }
19261 return Register();
19262}
19263
19264Register fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
19265 if (RetVT.SimpleTy != MVT::v2f64)
19266 return Register();
19267 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19268 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19269 }
19270 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19271 return fastEmitInst_rr(MachineInstOpcode: X86::UNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
19272 }
19273 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19274 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDrr, RC: &X86::VR128RegClass, Op0, Op1);
19275 }
19276 return Register();
19277}
19278
19279Register fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, Register Op0, Register Op1) {
19280 if (RetVT.SimpleTy != MVT::v4f64)
19281 return Register();
19282 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19283 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19284 }
19285 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
19286 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19287 }
19288 return Register();
19289}
19290
19291Register fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, Register Op0, Register Op1) {
19292 if (RetVT.SimpleTy != MVT::v8f64)
19293 return Register();
19294 if ((Subtarget->hasAVX512())) {
19295 return fastEmitInst_rr(MachineInstOpcode: X86::VUNPCKLPDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19296 }
19297 return Register();
19298}
19299
19300Register fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19301 switch (VT.SimpleTy) {
19302 case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op1);
19303 case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op1);
19304 case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op1);
19305 case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op1);
19306 case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op1);
19307 case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op1);
19308 case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op1);
19309 case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op1);
19310 case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op1);
19311 case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op1);
19312 case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op1);
19313 case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op1);
19314 case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op1);
19315 case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op1);
19316 case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op1);
19317 case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op1);
19318 case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op1);
19319 case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op1);
19320 default: return Register();
19321 }
19322}
19323
19324// FastEmit functions for X86ISD::VCVT2PH2BF8.
19325
19326Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19327 if (RetVT.SimpleTy != MVT::v16i8)
19328 return Register();
19329 if ((Subtarget->hasAVX10_2())) {
19330 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19331 }
19332 return Register();
19333}
19334
19335Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19336 if (RetVT.SimpleTy != MVT::v32i8)
19337 return Register();
19338 if ((Subtarget->hasAVX10_2())) {
19339 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19340 }
19341 return Register();
19342}
19343
19344Register fastEmit_X86ISD_VCVT2PH2BF8_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19345 if (RetVT.SimpleTy != MVT::v64i8)
19346 return Register();
19347 if ((Subtarget->hasAVX10_2())) {
19348 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19349 }
19350 return Register();
19351}
19352
19353Register fastEmit_X86ISD_VCVT2PH2BF8_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19354 switch (VT.SimpleTy) {
19355 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v8f16_rr(RetVT, Op0, Op1);
19356 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v16f16_rr(RetVT, Op0, Op1);
19357 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2BF8_MVT_v32f16_rr(RetVT, Op0, Op1);
19358 default: return Register();
19359 }
19360}
19361
19362// FastEmit functions for X86ISD::VCVT2PH2BF8S.
19363
19364Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19365 if (RetVT.SimpleTy != MVT::v16i8)
19366 return Register();
19367 if ((Subtarget->hasAVX10_2())) {
19368 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19369 }
19370 return Register();
19371}
19372
19373Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19374 if (RetVT.SimpleTy != MVT::v32i8)
19375 return Register();
19376 if ((Subtarget->hasAVX10_2())) {
19377 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19378 }
19379 return Register();
19380}
19381
19382Register fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19383 if (RetVT.SimpleTy != MVT::v64i8)
19384 return Register();
19385 if ((Subtarget->hasAVX10_2())) {
19386 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2BF8SZrr, RC: &X86::VR512RegClass, Op0, Op1);
19387 }
19388 return Register();
19389}
19390
19391Register fastEmit_X86ISD_VCVT2PH2BF8S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19392 switch (VT.SimpleTy) {
19393 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v8f16_rr(RetVT, Op0, Op1);
19394 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v16f16_rr(RetVT, Op0, Op1);
19395 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2BF8S_MVT_v32f16_rr(RetVT, Op0, Op1);
19396 default: return Register();
19397 }
19398}
19399
19400// FastEmit functions for X86ISD::VCVT2PH2HF8.
19401
19402Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19403 if (RetVT.SimpleTy != MVT::v16i8)
19404 return Register();
19405 if ((Subtarget->hasAVX10_2())) {
19406 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19407 }
19408 return Register();
19409}
19410
19411Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19412 if (RetVT.SimpleTy != MVT::v32i8)
19413 return Register();
19414 if ((Subtarget->hasAVX10_2())) {
19415 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19416 }
19417 return Register();
19418}
19419
19420Register fastEmit_X86ISD_VCVT2PH2HF8_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19421 if (RetVT.SimpleTy != MVT::v64i8)
19422 return Register();
19423 if ((Subtarget->hasAVX10_2())) {
19424 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19425 }
19426 return Register();
19427}
19428
19429Register fastEmit_X86ISD_VCVT2PH2HF8_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19430 switch (VT.SimpleTy) {
19431 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v8f16_rr(RetVT, Op0, Op1);
19432 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v16f16_rr(RetVT, Op0, Op1);
19433 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2HF8_MVT_v32f16_rr(RetVT, Op0, Op1);
19434 default: return Register();
19435 }
19436}
19437
19438// FastEmit functions for X86ISD::VCVT2PH2HF8S.
19439
19440Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
19441 if (RetVT.SimpleTy != MVT::v16i8)
19442 return Register();
19443 if ((Subtarget->hasAVX10_2())) {
19444 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19445 }
19446 return Register();
19447}
19448
19449Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v16f16_rr(MVT RetVT, Register Op0, Register Op1) {
19450 if (RetVT.SimpleTy != MVT::v32i8)
19451 return Register();
19452 if ((Subtarget->hasAVX10_2())) {
19453 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19454 }
19455 return Register();
19456}
19457
19458Register fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v32f16_rr(MVT RetVT, Register Op0, Register Op1) {
19459 if (RetVT.SimpleTy != MVT::v64i8)
19460 return Register();
19461 if ((Subtarget->hasAVX10_2())) {
19462 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PH2HF8SZrr, RC: &X86::VR512RegClass, Op0, Op1);
19463 }
19464 return Register();
19465}
19466
19467Register fastEmit_X86ISD_VCVT2PH2HF8S_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19468 switch (VT.SimpleTy) {
19469 case MVT::v8f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v8f16_rr(RetVT, Op0, Op1);
19470 case MVT::v16f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v16f16_rr(RetVT, Op0, Op1);
19471 case MVT::v32f16: return fastEmit_X86ISD_VCVT2PH2HF8S_MVT_v32f16_rr(RetVT, Op0, Op1);
19472 default: return Register();
19473 }
19474}
19475
19476// FastEmit functions for X86ISD::VFCMULC.
19477
19478Register fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19479 if (RetVT.SimpleTy != MVT::v4f32)
19480 return Register();
19481 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19482 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19483 }
19484 return Register();
19485}
19486
19487Register fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19488 if (RetVT.SimpleTy != MVT::v8f32)
19489 return Register();
19490 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19491 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19492 }
19493 return Register();
19494}
19495
19496Register fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19497 if (RetVT.SimpleTy != MVT::v16f32)
19498 return Register();
19499 if ((Subtarget->hasFP16())) {
19500 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
19501 }
19502 return Register();
19503}
19504
19505Register fastEmit_X86ISD_VFCMULC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19506 switch (VT.SimpleTy) {
19507 case MVT::v4f32: return fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
19508 case MVT::v8f32: return fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
19509 case MVT::v16f32: return fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
19510 default: return Register();
19511 }
19512}
19513
19514// FastEmit functions for X86ISD::VFCMULCSH.
19515
19516Register fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19517 if (RetVT.SimpleTy != MVT::v4f32)
19518 return Register();
19519 if ((Subtarget->hasFP16())) {
19520 return fastEmitInst_rr(MachineInstOpcode: X86::VFCMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
19521 }
19522 return Register();
19523}
19524
19525Register fastEmit_X86ISD_VFCMULCSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19526 switch (VT.SimpleTy) {
19527 case MVT::v4f32: return fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
19528 default: return Register();
19529 }
19530}
19531
19532// FastEmit functions for X86ISD::VFMULC.
19533
19534Register fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19535 if (RetVT.SimpleTy != MVT::v4f32)
19536 return Register();
19537 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19538 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19539 }
19540 return Register();
19541}
19542
19543Register fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19544 if (RetVT.SimpleTy != MVT::v8f32)
19545 return Register();
19546 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
19547 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19548 }
19549 return Register();
19550}
19551
19552Register fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19553 if (RetVT.SimpleTy != MVT::v16f32)
19554 return Register();
19555 if ((Subtarget->hasFP16())) {
19556 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCPHZrr, RC: &X86::VR512RegClass, Op0, Op1);
19557 }
19558 return Register();
19559}
19560
19561Register fastEmit_X86ISD_VFMULC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19562 switch (VT.SimpleTy) {
19563 case MVT::v4f32: return fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
19564 case MVT::v8f32: return fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
19565 case MVT::v16f32: return fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
19566 default: return Register();
19567 }
19568}
19569
19570// FastEmit functions for X86ISD::VFMULCSH.
19571
19572Register fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19573 if (RetVT.SimpleTy != MVT::v4f32)
19574 return Register();
19575 if ((Subtarget->hasFP16())) {
19576 return fastEmitInst_rr(MachineInstOpcode: X86::VFMULCSHZrr, RC: &X86::VR128XRegClass, Op0, Op1);
19577 }
19578 return Register();
19579}
19580
19581Register fastEmit_X86ISD_VFMULCSH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19582 switch (VT.SimpleTy) {
19583 case MVT::v4f32: return fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
19584 default: return Register();
19585 }
19586}
19587
19588// FastEmit functions for X86ISD::VFPROUND2.
19589
19590Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8f16_rr(Register Op0, Register Op1) {
19591 if ((Subtarget->hasAVX10_2())) {
19592 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19593 }
19594 return Register();
19595}
19596
19597Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8bf16_rr(Register Op0, Register Op1) {
19598 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
19599 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19600 }
19601 return Register();
19602}
19603
19604Register fastEmit_X86ISD_VFPROUND2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
19605switch (RetVT.SimpleTy) {
19606 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8f16_rr(Op0, Op1);
19607 case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_MVT_v8bf16_rr(Op0, Op1);
19608 default: return Register();
19609}
19610}
19611
19612Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16f16_rr(Register Op0, Register Op1) {
19613 if ((Subtarget->hasAVX10_2())) {
19614 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19615 }
19616 return Register();
19617}
19618
19619Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16bf16_rr(Register Op0, Register Op1) {
19620 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
19621 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Z256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19622 }
19623 return Register();
19624}
19625
19626Register fastEmit_X86ISD_VFPROUND2_MVT_v8f32_rr(MVT RetVT, Register Op0, Register Op1) {
19627switch (RetVT.SimpleTy) {
19628 case MVT::v16f16: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16f16_rr(Op0, Op1);
19629 case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_MVT_v16bf16_rr(Op0, Op1);
19630 default: return Register();
19631}
19632}
19633
19634Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32f16_rr(Register Op0, Register Op1) {
19635 if ((Subtarget->hasAVX10_2())) {
19636 return fastEmitInst_rr(MachineInstOpcode: X86::VCVT2PS2PHXZrr, RC: &X86::VR512RegClass, Op0, Op1);
19637 }
19638 return Register();
19639}
19640
19641Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32bf16_rr(Register Op0, Register Op1) {
19642 if ((Subtarget->hasBF16())) {
19643 return fastEmitInst_rr(MachineInstOpcode: X86::VCVTNE2PS2BF16Zrr, RC: &X86::VR512RegClass, Op0, Op1);
19644 }
19645 return Register();
19646}
19647
19648Register fastEmit_X86ISD_VFPROUND2_MVT_v16f32_rr(MVT RetVT, Register Op0, Register Op1) {
19649switch (RetVT.SimpleTy) {
19650 case MVT::v32f16: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32f16_rr(Op0, Op1);
19651 case MVT::v32bf16: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_MVT_v32bf16_rr(Op0, Op1);
19652 default: return Register();
19653}
19654}
19655
19656Register fastEmit_X86ISD_VFPROUND2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19657 switch (VT.SimpleTy) {
19658 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND2_MVT_v4f32_rr(RetVT, Op0, Op1);
19659 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND2_MVT_v8f32_rr(RetVT, Op0, Op1);
19660 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND2_MVT_v16f32_rr(RetVT, Op0, Op1);
19661 default: return Register();
19662 }
19663}
19664
19665// FastEmit functions for X86ISD::VP2INTERSECT.
19666
19667Register fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19668 if (RetVT.SimpleTy != MVT::Untyped)
19669 return Register();
19670 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19671 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ128rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
19672 }
19673 return Register();
19674}
19675
19676Register fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19677 if (RetVT.SimpleTy != MVT::Untyped)
19678 return Register();
19679 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19680 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZ256rr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
19681 }
19682 return Register();
19683}
19684
19685Register fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19686 if (RetVT.SimpleTy != MVT::Untyped)
19687 return Register();
19688 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
19689 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTDZrr, RC: &X86::VK16PAIRRegClass, Op0, Op1);
19690 }
19691 return Register();
19692}
19693
19694Register fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19695 if (RetVT.SimpleTy != MVT::Untyped)
19696 return Register();
19697 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19698 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ128rr, RC: &X86::VK2PAIRRegClass, Op0, Op1);
19699 }
19700 return Register();
19701}
19702
19703Register fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19704 if (RetVT.SimpleTy != MVT::Untyped)
19705 return Register();
19706 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
19707 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZ256rr, RC: &X86::VK4PAIRRegClass, Op0, Op1);
19708 }
19709 return Register();
19710}
19711
19712Register fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19713 if (RetVT.SimpleTy != MVT::Untyped)
19714 return Register();
19715 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
19716 return fastEmitInst_rr(MachineInstOpcode: X86::VP2INTERSECTQZrr, RC: &X86::VK8PAIRRegClass, Op0, Op1);
19717 }
19718 return Register();
19719}
19720
19721Register fastEmit_X86ISD_VP2INTERSECT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19722 switch (VT.SimpleTy) {
19723 case MVT::v4i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(RetVT, Op0, Op1);
19724 case MVT::v8i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(RetVT, Op0, Op1);
19725 case MVT::v16i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(RetVT, Op0, Op1);
19726 case MVT::v2i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(RetVT, Op0, Op1);
19727 case MVT::v4i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(RetVT, Op0, Op1);
19728 case MVT::v8i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(RetVT, Op0, Op1);
19729 default: return Register();
19730 }
19731}
19732
19733// FastEmit functions for X86ISD::VPERMV.
19734
19735Register fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19736 if (RetVT.SimpleTy != MVT::v16i8)
19737 return Register();
19738 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
19739 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19740 }
19741 return Register();
19742}
19743
19744Register fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19745 if (RetVT.SimpleTy != MVT::v32i8)
19746 return Register();
19747 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
19748 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19749 }
19750 return Register();
19751}
19752
19753Register fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19754 if (RetVT.SimpleTy != MVT::v64i8)
19755 return Register();
19756 if ((Subtarget->hasVBMI())) {
19757 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMBZrr, RC: &X86::VR512RegClass, Op0, Op1);
19758 }
19759 return Register();
19760}
19761
19762Register fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19763 if (RetVT.SimpleTy != MVT::v8i16)
19764 return Register();
19765 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19766 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19767 }
19768 return Register();
19769}
19770
19771Register fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19772 if (RetVT.SimpleTy != MVT::v16i16)
19773 return Register();
19774 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19775 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19776 }
19777 return Register();
19778}
19779
19780Register fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19781 if (RetVT.SimpleTy != MVT::v32i16)
19782 return Register();
19783 if ((Subtarget->hasBWI())) {
19784 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19785 }
19786 return Register();
19787}
19788
19789Register fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
19790 if (RetVT.SimpleTy != MVT::v8i32)
19791 return Register();
19792 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19793 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19794 }
19795 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
19796 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19797 }
19798 return Register();
19799}
19800
19801Register fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
19802 if (RetVT.SimpleTy != MVT::v16i32)
19803 return Register();
19804 if ((Subtarget->hasAVX512())) {
19805 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19806 }
19807 return Register();
19808}
19809
19810Register fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
19811 if (RetVT.SimpleTy != MVT::v4i64)
19812 return Register();
19813 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
19814 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19815 }
19816 return Register();
19817}
19818
19819Register fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
19820 if (RetVT.SimpleTy != MVT::v8i64)
19821 return Register();
19822 if ((Subtarget->hasAVX512())) {
19823 return fastEmitInst_rr(MachineInstOpcode: X86::VPERMQZrr, RC: &X86::VR512RegClass, Op0, Op1);
19824 }
19825 return Register();
19826}
19827
19828Register fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19829 switch (VT.SimpleTy) {
19830 case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op1);
19831 case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op1);
19832 case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op1);
19833 case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op1);
19834 case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op1);
19835 case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op1);
19836 case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op1);
19837 case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op1);
19838 case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op1);
19839 case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op1);
19840 default: return Register();
19841 }
19842}
19843
19844// FastEmit functions for X86ISD::VPMADDUBSW.
19845
19846Register fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19847 if (RetVT.SimpleTy != MVT::v8i16)
19848 return Register();
19849 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19850 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19851 }
19852 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
19853 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
19854 }
19855 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19856 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWrr, RC: &X86::VR128RegClass, Op0, Op1);
19857 }
19858 return Register();
19859}
19860
19861Register fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
19862 if (RetVT.SimpleTy != MVT::v16i16)
19863 return Register();
19864 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19865 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19866 }
19867 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19868 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWYrr, RC: &X86::VR256RegClass, Op0, Op1);
19869 }
19870 return Register();
19871}
19872
19873Register fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
19874 if (RetVT.SimpleTy != MVT::v32i16)
19875 return Register();
19876 if ((Subtarget->hasBWI())) {
19877 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDUBSWZrr, RC: &X86::VR512RegClass, Op0, Op1);
19878 }
19879 return Register();
19880}
19881
19882Register fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19883 switch (VT.SimpleTy) {
19884 case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op1);
19885 case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op1);
19886 case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op1);
19887 default: return Register();
19888 }
19889}
19890
19891// FastEmit functions for X86ISD::VPMADDWD.
19892
19893Register fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19894 if (RetVT.SimpleTy != MVT::v4i32)
19895 return Register();
19896 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19897 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
19898 }
19899 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
19900 return fastEmitInst_rr(MachineInstOpcode: X86::PMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19901 }
19902 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19903 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDrr, RC: &X86::VR128RegClass, Op0, Op1);
19904 }
19905 return Register();
19906}
19907
19908Register fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
19909 if (RetVT.SimpleTy != MVT::v8i32)
19910 return Register();
19911 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
19912 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
19913 }
19914 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
19915 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDYrr, RC: &X86::VR256RegClass, Op0, Op1);
19916 }
19917 return Register();
19918}
19919
19920Register fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
19921 if (RetVT.SimpleTy != MVT::v16i32)
19922 return Register();
19923 if ((Subtarget->hasBWI())) {
19924 return fastEmitInst_rr(MachineInstOpcode: X86::VPMADDWDZrr, RC: &X86::VR512RegClass, Op0, Op1);
19925 }
19926 return Register();
19927}
19928
19929Register fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19930 switch (VT.SimpleTy) {
19931 case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op1);
19932 case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op1);
19933 case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op1);
19934 default: return Register();
19935 }
19936}
19937
19938// FastEmit functions for X86ISD::VPSHA.
19939
19940Register fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19941 if (RetVT.SimpleTy != MVT::v16i8)
19942 return Register();
19943 if ((Subtarget->hasXOP())) {
19944 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHABrr, RC: &X86::VR128RegClass, Op0, Op1);
19945 }
19946 return Register();
19947}
19948
19949Register fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19950 if (RetVT.SimpleTy != MVT::v8i16)
19951 return Register();
19952 if ((Subtarget->hasXOP())) {
19953 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAWrr, RC: &X86::VR128RegClass, Op0, Op1);
19954 }
19955 return Register();
19956}
19957
19958Register fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
19959 if (RetVT.SimpleTy != MVT::v4i32)
19960 return Register();
19961 if ((Subtarget->hasXOP())) {
19962 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHADrr, RC: &X86::VR128RegClass, Op0, Op1);
19963 }
19964 return Register();
19965}
19966
19967Register fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
19968 if (RetVT.SimpleTy != MVT::v2i64)
19969 return Register();
19970 if ((Subtarget->hasXOP())) {
19971 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHAQrr, RC: &X86::VR128RegClass, Op0, Op1);
19972 }
19973 return Register();
19974}
19975
19976Register fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
19977 switch (VT.SimpleTy) {
19978 case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op1);
19979 case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op1);
19980 case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op1);
19981 case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op1);
19982 default: return Register();
19983 }
19984}
19985
19986// FastEmit functions for X86ISD::VPSHL.
19987
19988Register fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
19989 if (RetVT.SimpleTy != MVT::v16i8)
19990 return Register();
19991 if ((Subtarget->hasXOP())) {
19992 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLBrr, RC: &X86::VR128RegClass, Op0, Op1);
19993 }
19994 return Register();
19995}
19996
19997Register fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
19998 if (RetVT.SimpleTy != MVT::v8i16)
19999 return Register();
20000 if ((Subtarget->hasXOP())) {
20001 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20002 }
20003 return Register();
20004}
20005
20006Register fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20007 if (RetVT.SimpleTy != MVT::v4i32)
20008 return Register();
20009 if ((Subtarget->hasXOP())) {
20010 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20011 }
20012 return Register();
20013}
20014
20015Register fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20016 if (RetVT.SimpleTy != MVT::v2i64)
20017 return Register();
20018 if ((Subtarget->hasXOP())) {
20019 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20020 }
20021 return Register();
20022}
20023
20024Register fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20025 switch (VT.SimpleTy) {
20026 case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op1);
20027 case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
20028 case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
20029 case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
20030 default: return Register();
20031 }
20032}
20033
20034// FastEmit functions for X86ISD::VPSHUFBITQMB.
20035
20036Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
20037 if (RetVT.SimpleTy != MVT::v16i1)
20038 return Register();
20039 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
20040 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ128rr, RC: &X86::VK16RegClass, Op0, Op1);
20041 }
20042 return Register();
20043}
20044
20045Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, Register Op0, Register Op1) {
20046 if (RetVT.SimpleTy != MVT::v32i1)
20047 return Register();
20048 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
20049 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZ256rr, RC: &X86::VK32RegClass, Op0, Op1);
20050 }
20051 return Register();
20052}
20053
20054Register fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, Register Op0, Register Op1) {
20055 if (RetVT.SimpleTy != MVT::v64i1)
20056 return Register();
20057 if ((Subtarget->hasBITALG())) {
20058 return fastEmitInst_rr(MachineInstOpcode: X86::VPSHUFBITQMBZrr, RC: &X86::VK64RegClass, Op0, Op1);
20059 }
20060 return Register();
20061}
20062
20063Register fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20064 switch (VT.SimpleTy) {
20065 case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op1);
20066 case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op1);
20067 case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op1);
20068 default: return Register();
20069 }
20070}
20071
20072// FastEmit functions for X86ISD::VSHL.
20073
20074Register fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20075 if (RetVT.SimpleTy != MVT::v8i16)
20076 return Register();
20077 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20078 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20079 }
20080 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20081 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20082 }
20083 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
20084 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20085 }
20086 return Register();
20087}
20088
20089Register fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20090 if (RetVT.SimpleTy != MVT::v4i32)
20091 return Register();
20092 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20093 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20094 }
20095 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20096 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20097 }
20098 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20099 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20100 }
20101 return Register();
20102}
20103
20104Register fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20105 if (RetVT.SimpleTy != MVT::v2i64)
20106 return Register();
20107 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20108 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20109 }
20110 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20111 return fastEmitInst_rr(MachineInstOpcode: X86::PSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20112 }
20113 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20114 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20115 }
20116 return Register();
20117}
20118
20119Register fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20120 switch (VT.SimpleTy) {
20121 case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
20122 case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
20123 case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
20124 default: return Register();
20125 }
20126}
20127
20128// FastEmit functions for X86ISD::VSHLV.
20129
20130Register fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20131 if (RetVT.SimpleTy != MVT::v8i16)
20132 return Register();
20133 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20134 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20135 }
20136 return Register();
20137}
20138
20139Register fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20140 if (RetVT.SimpleTy != MVT::v16i16)
20141 return Register();
20142 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20143 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20144 }
20145 return Register();
20146}
20147
20148Register fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20149 if (RetVT.SimpleTy != MVT::v32i16)
20150 return Register();
20151 if ((Subtarget->hasBWI())) {
20152 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20153 }
20154 return Register();
20155}
20156
20157Register fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20158 if (RetVT.SimpleTy != MVT::v4i32)
20159 return Register();
20160 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20161 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20162 }
20163 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20164 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20165 }
20166 return Register();
20167}
20168
20169Register fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20170 if (RetVT.SimpleTy != MVT::v8i32)
20171 return Register();
20172 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20173 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20174 }
20175 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20176 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20177 }
20178 return Register();
20179}
20180
20181Register fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20182 if (RetVT.SimpleTy != MVT::v16i32)
20183 return Register();
20184 if ((Subtarget->hasAVX512())) {
20185 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20186 }
20187 return Register();
20188}
20189
20190Register fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20191 if (RetVT.SimpleTy != MVT::v2i64)
20192 return Register();
20193 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20194 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20195 }
20196 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20197 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
20198 }
20199 return Register();
20200}
20201
20202Register fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20203 if (RetVT.SimpleTy != MVT::v4i64)
20204 return Register();
20205 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20206 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20207 }
20208 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20209 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
20210 }
20211 return Register();
20212}
20213
20214Register fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20215 if (RetVT.SimpleTy != MVT::v8i64)
20216 return Register();
20217 if ((Subtarget->hasAVX512())) {
20218 return fastEmitInst_rr(MachineInstOpcode: X86::VPSLLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20219 }
20220 return Register();
20221}
20222
20223Register fastEmit_X86ISD_VSHLV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20224 switch (VT.SimpleTy) {
20225 case MVT::v8i16: return fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(RetVT, Op0, Op1);
20226 case MVT::v16i16: return fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(RetVT, Op0, Op1);
20227 case MVT::v32i16: return fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(RetVT, Op0, Op1);
20228 case MVT::v4i32: return fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(RetVT, Op0, Op1);
20229 case MVT::v8i32: return fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(RetVT, Op0, Op1);
20230 case MVT::v16i32: return fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(RetVT, Op0, Op1);
20231 case MVT::v2i64: return fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(RetVT, Op0, Op1);
20232 case MVT::v4i64: return fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(RetVT, Op0, Op1);
20233 case MVT::v8i64: return fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(RetVT, Op0, Op1);
20234 default: return Register();
20235 }
20236}
20237
20238// FastEmit functions for X86ISD::VSRA.
20239
20240Register fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20241 if (RetVT.SimpleTy != MVT::v8i16)
20242 return Register();
20243 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20244 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20245 }
20246 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20247 return fastEmitInst_rr(MachineInstOpcode: X86::PSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
20248 }
20249 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
20250 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAWrr, RC: &X86::VR128RegClass, Op0, Op1);
20251 }
20252 return Register();
20253}
20254
20255Register fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20256 if (RetVT.SimpleTy != MVT::v4i32)
20257 return Register();
20258 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20259 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20260 }
20261 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20262 return fastEmitInst_rr(MachineInstOpcode: X86::PSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
20263 }
20264 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20265 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRADrr, RC: &X86::VR128RegClass, Op0, Op1);
20266 }
20267 return Register();
20268}
20269
20270Register fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20271 if (RetVT.SimpleTy != MVT::v2i64)
20272 return Register();
20273 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20274 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20275 }
20276 return Register();
20277}
20278
20279Register fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20280 switch (VT.SimpleTy) {
20281 case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op1);
20282 case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op1);
20283 case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op1);
20284 default: return Register();
20285 }
20286}
20287
20288// FastEmit functions for X86ISD::VSRAV.
20289
20290Register fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20291 if (RetVT.SimpleTy != MVT::v8i16)
20292 return Register();
20293 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20294 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20295 }
20296 return Register();
20297}
20298
20299Register fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20300 if (RetVT.SimpleTy != MVT::v16i16)
20301 return Register();
20302 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20303 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20304 }
20305 return Register();
20306}
20307
20308Register fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20309 if (RetVT.SimpleTy != MVT::v32i16)
20310 return Register();
20311 if ((Subtarget->hasBWI())) {
20312 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20313 }
20314 return Register();
20315}
20316
20317Register fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20318 if (RetVT.SimpleTy != MVT::v4i32)
20319 return Register();
20320 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20321 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20322 }
20323 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20324 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20325 }
20326 return Register();
20327}
20328
20329Register fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20330 if (RetVT.SimpleTy != MVT::v8i32)
20331 return Register();
20332 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20333 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20334 }
20335 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20336 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20337 }
20338 return Register();
20339}
20340
20341Register fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20342 if (RetVT.SimpleTy != MVT::v16i32)
20343 return Register();
20344 if ((Subtarget->hasAVX512())) {
20345 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20346 }
20347 return Register();
20348}
20349
20350Register fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20351 if (RetVT.SimpleTy != MVT::v2i64)
20352 return Register();
20353 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20354 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20355 }
20356 return Register();
20357}
20358
20359Register fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20360 if (RetVT.SimpleTy != MVT::v4i64)
20361 return Register();
20362 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20363 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20364 }
20365 return Register();
20366}
20367
20368Register fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20369 if (RetVT.SimpleTy != MVT::v8i64)
20370 return Register();
20371 if ((Subtarget->hasAVX512())) {
20372 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRAVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20373 }
20374 return Register();
20375}
20376
20377Register fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20378 switch (VT.SimpleTy) {
20379 case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op1);
20380 case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op1);
20381 case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op1);
20382 case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op1);
20383 case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op1);
20384 case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op1);
20385 case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op1);
20386 case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op1);
20387 case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op1);
20388 default: return Register();
20389 }
20390}
20391
20392// FastEmit functions for X86ISD::VSRL.
20393
20394Register fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20395 if (RetVT.SimpleTy != MVT::v8i16)
20396 return Register();
20397 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20398 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20399 }
20400 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20401 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20402 }
20403 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
20404 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLWrr, RC: &X86::VR128RegClass, Op0, Op1);
20405 }
20406 return Register();
20407}
20408
20409Register fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20410 if (RetVT.SimpleTy != MVT::v4i32)
20411 return Register();
20412 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20413 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20414 }
20415 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20416 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20417 }
20418 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20419 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLDrr, RC: &X86::VR128RegClass, Op0, Op1);
20420 }
20421 return Register();
20422}
20423
20424Register fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20425 if (RetVT.SimpleTy != MVT::v2i64)
20426 return Register();
20427 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20428 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20429 }
20430 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
20431 return fastEmitInst_rr(MachineInstOpcode: X86::PSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20432 }
20433 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
20434 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLQrr, RC: &X86::VR128RegClass, Op0, Op1);
20435 }
20436 return Register();
20437}
20438
20439Register fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20440 switch (VT.SimpleTy) {
20441 case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op1);
20442 case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op1);
20443 case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op1);
20444 default: return Register();
20445 }
20446}
20447
20448// FastEmit functions for X86ISD::VSRLV.
20449
20450Register fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
20451 if (RetVT.SimpleTy != MVT::v8i16)
20452 return Register();
20453 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20454 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20455 }
20456 return Register();
20457}
20458
20459Register fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(MVT RetVT, Register Op0, Register Op1) {
20460 if (RetVT.SimpleTy != MVT::v16i16)
20461 return Register();
20462 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
20463 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20464 }
20465 return Register();
20466}
20467
20468Register fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(MVT RetVT, Register Op0, Register Op1) {
20469 if (RetVT.SimpleTy != MVT::v32i16)
20470 return Register();
20471 if ((Subtarget->hasBWI())) {
20472 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVWZrr, RC: &X86::VR512RegClass, Op0, Op1);
20473 }
20474 return Register();
20475}
20476
20477Register fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
20478 if (RetVT.SimpleTy != MVT::v4i32)
20479 return Register();
20480 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20481 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20482 }
20483 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20484 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDrr, RC: &X86::VR128RegClass, Op0, Op1);
20485 }
20486 return Register();
20487}
20488
20489Register fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(MVT RetVT, Register Op0, Register Op1) {
20490 if (RetVT.SimpleTy != MVT::v8i32)
20491 return Register();
20492 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20493 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20494 }
20495 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20496 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDYrr, RC: &X86::VR256RegClass, Op0, Op1);
20497 }
20498 return Register();
20499}
20500
20501Register fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(MVT RetVT, Register Op0, Register Op1) {
20502 if (RetVT.SimpleTy != MVT::v16i32)
20503 return Register();
20504 if ((Subtarget->hasAVX512())) {
20505 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVDZrr, RC: &X86::VR512RegClass, Op0, Op1);
20506 }
20507 return Register();
20508}
20509
20510Register fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
20511 if (RetVT.SimpleTy != MVT::v2i64)
20512 return Register();
20513 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20514 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ128rr, RC: &X86::VR128XRegClass, Op0, Op1);
20515 }
20516 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20517 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQrr, RC: &X86::VR128RegClass, Op0, Op1);
20518 }
20519 return Register();
20520}
20521
20522Register fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(MVT RetVT, Register Op0, Register Op1) {
20523 if (RetVT.SimpleTy != MVT::v4i64)
20524 return Register();
20525 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
20526 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZ256rr, RC: &X86::VR256XRegClass, Op0, Op1);
20527 }
20528 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
20529 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQYrr, RC: &X86::VR256RegClass, Op0, Op1);
20530 }
20531 return Register();
20532}
20533
20534Register fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(MVT RetVT, Register Op0, Register Op1) {
20535 if (RetVT.SimpleTy != MVT::v8i64)
20536 return Register();
20537 if ((Subtarget->hasAVX512())) {
20538 return fastEmitInst_rr(MachineInstOpcode: X86::VPSRLVQZrr, RC: &X86::VR512RegClass, Op0, Op1);
20539 }
20540 return Register();
20541}
20542
20543Register fastEmit_X86ISD_VSRLV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
20544 switch (VT.SimpleTy) {
20545 case MVT::v8i16: return fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(RetVT, Op0, Op1);
20546 case MVT::v16i16: return fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(RetVT, Op0, Op1);
20547 case MVT::v32i16: return fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(RetVT, Op0, Op1);
20548 case MVT::v4i32: return fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(RetVT, Op0, Op1);
20549 case MVT::v8i32: return fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(RetVT, Op0, Op1);
20550 case MVT::v16i32: return fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(RetVT, Op0, Op1);
20551 case MVT::v2i64: return fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(RetVT, Op0, Op1);
20552 case MVT::v4i64: return fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(RetVT, Op0, Op1);
20553 case MVT::v8i64: return fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(RetVT, Op0, Op1);
20554 default: return Register();
20555 }
20556}
20557
20558// Top-level FastEmit function.
20559
20560Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
20561 switch (Opcode) {
20562 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
20563 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
20564 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
20565 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
20566 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
20567 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
20568 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
20569 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
20570 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
20571 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
20572 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
20573 case ISD::PDEP: return fastEmit_ISD_PDEP_rr(VT, RetVT, Op0, Op1);
20574 case ISD::PEXT: return fastEmit_ISD_PEXT_rr(VT, RetVT, Op0, Op1);
20575 case ISD::PSEUDO_FMAX: return fastEmit_ISD_PSEUDO_FMAX_rr(VT, RetVT, Op0, Op1);
20576 case ISD::PSEUDO_FMIN: return fastEmit_ISD_PSEUDO_FMIN_rr(VT, RetVT, Op0, Op1);
20577 case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op1);
20578 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
20579 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
20580 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
20581 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
20582 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
20583 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
20584 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
20585 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
20586 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
20587 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
20588 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
20589 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
20590 case ISD::STRICT_PSEUDO_FMAX: return fastEmit_ISD_STRICT_PSEUDO_FMAX_rr(VT, RetVT, Op0, Op1);
20591 case ISD::STRICT_PSEUDO_FMIN: return fastEmit_ISD_STRICT_PSEUDO_FMIN_rr(VT, RetVT, Op0, Op1);
20592 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
20593 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
20594 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
20595 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
20596 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
20597 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
20598 case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op1);
20599 case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op1);
20600 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op1);
20601 case X86ISD::BT: return fastEmit_X86ISD_BT_rr(VT, RetVT, Op0, Op1);
20602 case X86ISD::BZHI: return fastEmit_X86ISD_BZHI_rr(VT, RetVT, Op0, Op1);
20603 case X86ISD::CMP: return fastEmit_X86ISD_CMP_rr(VT, RetVT, Op0, Op1);
20604 case X86ISD::COMI: return fastEmit_X86ISD_COMI_rr(VT, RetVT, Op0, Op1);
20605 case X86ISD::COMX: return fastEmit_X86ISD_COMX_rr(VT, RetVT, Op0, Op1);
20606 case X86ISD::FADDS: return fastEmit_X86ISD_FADDS_rr(VT, RetVT, Op0, Op1);
20607 case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op1);
20608 case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op1);
20609 case X86ISD::FCMP: return fastEmit_X86ISD_FCMP_rr(VT, RetVT, Op0, Op1);
20610 case X86ISD::FDIVS: return fastEmit_X86ISD_FDIVS_rr(VT, RetVT, Op0, Op1);
20611 case X86ISD::FGETEXPS: return fastEmit_X86ISD_FGETEXPS_rr(VT, RetVT, Op0, Op1);
20612 case X86ISD::FGETEXPS_SAE: return fastEmit_X86ISD_FGETEXPS_SAE_rr(VT, RetVT, Op0, Op1);
20613 case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op1);
20614 case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op1);
20615 case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op1);
20616 case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op1);
20617 case X86ISD::FMAXS_SAE: return fastEmit_X86ISD_FMAXS_SAE_rr(VT, RetVT, Op0, Op1);
20618 case X86ISD::FMAX_SAE: return fastEmit_X86ISD_FMAX_SAE_rr(VT, RetVT, Op0, Op1);
20619 case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op1);
20620 case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op1);
20621 case X86ISD::FMINS_SAE: return fastEmit_X86ISD_FMINS_SAE_rr(VT, RetVT, Op0, Op1);
20622 case X86ISD::FMIN_SAE: return fastEmit_X86ISD_FMIN_SAE_rr(VT, RetVT, Op0, Op1);
20623 case X86ISD::FMULS: return fastEmit_X86ISD_FMULS_rr(VT, RetVT, Op0, Op1);
20624 case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op1);
20625 case X86ISD::FP80_ADD: return fastEmit_X86ISD_FP80_ADD_rr(VT, RetVT, Op0, Op1);
20626 case X86ISD::FSQRTS: return fastEmit_X86ISD_FSQRTS_rr(VT, RetVT, Op0, Op1);
20627 case X86ISD::FSUBS: return fastEmit_X86ISD_FSUBS_rr(VT, RetVT, Op0, Op1);
20628 case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op1);
20629 case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op1);
20630 case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op1);
20631 case X86ISD::HADDS: return fastEmit_X86ISD_HADDS_rr(VT, RetVT, Op0, Op1);
20632 case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op1);
20633 case X86ISD::HSUBS: return fastEmit_X86ISD_HSUBS_rr(VT, RetVT, Op0, Op1);
20634 case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op1);
20635 case X86ISD::KORTEST: return fastEmit_X86ISD_KORTEST_rr(VT, RetVT, Op0, Op1);
20636 case X86ISD::KTEST: return fastEmit_X86ISD_KTEST_rr(VT, RetVT, Op0, Op1);
20637 case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op1);
20638 case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op1);
20639 case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op1);
20640 case X86ISD::MOVSH: return fastEmit_X86ISD_MOVSH_rr(VT, RetVT, Op0, Op1);
20641 case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op1);
20642 case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op1);
20643 case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op1);
20644 case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op1);
20645 case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op1);
20646 case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op1);
20647 case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op1);
20648 case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op1);
20649 case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op1);
20650 case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op1);
20651 case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op1);
20652 case X86ISD::PTEST: return fastEmit_X86ISD_PTEST_rr(VT, RetVT, Op0, Op1);
20653 case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op1);
20654 case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op1);
20655 case X86ISD::SCALEF: return fastEmit_X86ISD_SCALEF_rr(VT, RetVT, Op0, Op1);
20656 case X86ISD::SCALEFS: return fastEmit_X86ISD_SCALEFS_rr(VT, RetVT, Op0, Op1);
20657 case X86ISD::STRICT_FCMP: return fastEmit_X86ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
20658 case X86ISD::STRICT_FCMPS: return fastEmit_X86ISD_STRICT_FCMPS_rr(VT, RetVT, Op0, Op1);
20659 case X86ISD::STRICT_FP80_ADD: return fastEmit_X86ISD_STRICT_FP80_ADD_rr(VT, RetVT, Op0, Op1);
20660 case X86ISD::TESTP: return fastEmit_X86ISD_TESTP_rr(VT, RetVT, Op0, Op1);
20661 case X86ISD::UCOMI: return fastEmit_X86ISD_UCOMI_rr(VT, RetVT, Op0, Op1);
20662 case X86ISD::UCOMX: return fastEmit_X86ISD_UCOMX_rr(VT, RetVT, Op0, Op1);
20663 case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op1);
20664 case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op1);
20665 case X86ISD::VCVT2PH2BF8: return fastEmit_X86ISD_VCVT2PH2BF8_rr(VT, RetVT, Op0, Op1);
20666 case X86ISD::VCVT2PH2BF8S: return fastEmit_X86ISD_VCVT2PH2BF8S_rr(VT, RetVT, Op0, Op1);
20667 case X86ISD::VCVT2PH2HF8: return fastEmit_X86ISD_VCVT2PH2HF8_rr(VT, RetVT, Op0, Op1);
20668 case X86ISD::VCVT2PH2HF8S: return fastEmit_X86ISD_VCVT2PH2HF8S_rr(VT, RetVT, Op0, Op1);
20669 case X86ISD::VFCMULC: return fastEmit_X86ISD_VFCMULC_rr(VT, RetVT, Op0, Op1);
20670 case X86ISD::VFCMULCSH: return fastEmit_X86ISD_VFCMULCSH_rr(VT, RetVT, Op0, Op1);
20671 case X86ISD::VFMULC: return fastEmit_X86ISD_VFMULC_rr(VT, RetVT, Op0, Op1);
20672 case X86ISD::VFMULCSH: return fastEmit_X86ISD_VFMULCSH_rr(VT, RetVT, Op0, Op1);
20673 case X86ISD::VFPROUND2: return fastEmit_X86ISD_VFPROUND2_rr(VT, RetVT, Op0, Op1);
20674 case X86ISD::VP2INTERSECT: return fastEmit_X86ISD_VP2INTERSECT_rr(VT, RetVT, Op0, Op1);
20675 case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op1);
20676 case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op1);
20677 case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op1);
20678 case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op1);
20679 case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op1);
20680 case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op1);
20681 case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op1);
20682 case X86ISD::VSHLV: return fastEmit_X86ISD_VSHLV_rr(VT, RetVT, Op0, Op1);
20683 case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op1);
20684 case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op1);
20685 case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op1);
20686 case X86ISD::VSRLV: return fastEmit_X86ISD_VSRLV_rr(VT, RetVT, Op0, Op1);
20687 default: return Register();
20688 }
20689}
20690
20691// FastEmit functions for ISD::ADD.
20692
20693Register fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20694 if (RetVT.SimpleTy != MVT::i8)
20695 return Register();
20696 if ((Subtarget->hasNDD())) {
20697 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20698 }
20699 if ((!Subtarget->hasNDD())) {
20700 return fastEmitInst_ri(MachineInstOpcode: X86::ADD8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20701 }
20702 return Register();
20703}
20704
20705Register fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20706 if (RetVT.SimpleTy != MVT::i16)
20707 return Register();
20708 if ((Subtarget->hasNDD())) {
20709 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20710 }
20711 if ((!Subtarget->hasNDD())) {
20712 return fastEmitInst_ri(MachineInstOpcode: X86::ADD16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20713 }
20714 return Register();
20715}
20716
20717Register fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20718 if (RetVT.SimpleTy != MVT::i32)
20719 return Register();
20720 if ((Subtarget->hasNDD())) {
20721 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20722 }
20723 if ((!Subtarget->hasNDD())) {
20724 return fastEmitInst_ri(MachineInstOpcode: X86::ADD32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20725 }
20726 return Register();
20727}
20728
20729Register fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20730 switch (VT.SimpleTy) {
20731 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, imm1);
20732 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, imm1);
20733 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, imm1);
20734 default: return Register();
20735 }
20736}
20737
20738// FastEmit functions for ISD::AND.
20739
20740Register fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20741 if (RetVT.SimpleTy != MVT::i8)
20742 return Register();
20743 if ((Subtarget->hasNDD())) {
20744 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20745 }
20746 if ((!Subtarget->hasNDD())) {
20747 return fastEmitInst_ri(MachineInstOpcode: X86::AND8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20748 }
20749 return Register();
20750}
20751
20752Register fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20753 if (RetVT.SimpleTy != MVT::i16)
20754 return Register();
20755 if ((Subtarget->hasNDD())) {
20756 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20757 }
20758 if ((!Subtarget->hasNDD())) {
20759 return fastEmitInst_ri(MachineInstOpcode: X86::AND16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20760 }
20761 return Register();
20762}
20763
20764Register fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20765 if (RetVT.SimpleTy != MVT::i32)
20766 return Register();
20767 if ((Subtarget->hasNDD())) {
20768 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20769 }
20770 if ((!Subtarget->hasNDD())) {
20771 return fastEmitInst_ri(MachineInstOpcode: X86::AND32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20772 }
20773 return Register();
20774}
20775
20776Register fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20777 switch (VT.SimpleTy) {
20778 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, imm1);
20779 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, imm1);
20780 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, imm1);
20781 default: return Register();
20782 }
20783}
20784
20785// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
20786
20787Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20788 if (RetVT.SimpleTy != MVT::i32)
20789 return Register();
20790 if ((Subtarget->hasDQI())) {
20791 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDZrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20792 }
20793 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
20794 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRDrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20795 }
20796 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
20797 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRDrri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20798 }
20799 return Register();
20800}
20801
20802Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20803 if (RetVT.SimpleTy != MVT::i64)
20804 return Register();
20805 if ((Subtarget->hasDQI())) {
20806 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQZrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20807 }
20808 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
20809 return fastEmitInst_ri(MachineInstOpcode: X86::PEXTRQrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20810 }
20811 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
20812 return fastEmitInst_ri(MachineInstOpcode: X86::VPEXTRQrri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20813 }
20814 return Register();
20815}
20816
20817Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20818 switch (VT.SimpleTy) {
20819 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, imm1);
20820 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, imm1);
20821 default: return Register();
20822 }
20823}
20824
20825// FastEmit functions for ISD::MUL.
20826
20827Register fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20828 if (RetVT.SimpleTy != MVT::i16)
20829 return Register();
20830 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL16rri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20831}
20832
20833Register fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20834 if (RetVT.SimpleTy != MVT::i32)
20835 return Register();
20836 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL32rri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20837}
20838
20839Register fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20840 switch (VT.SimpleTy) {
20841 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, imm1);
20842 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, imm1);
20843 default: return Register();
20844 }
20845}
20846
20847// FastEmit functions for ISD::OR.
20848
20849Register fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20850 if (RetVT.SimpleTy != MVT::i8)
20851 return Register();
20852 if ((Subtarget->hasNDD())) {
20853 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20854 }
20855 if ((!Subtarget->hasNDD())) {
20856 return fastEmitInst_ri(MachineInstOpcode: X86::OR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20857 }
20858 return Register();
20859}
20860
20861Register fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20862 if (RetVT.SimpleTy != MVT::i16)
20863 return Register();
20864 if ((Subtarget->hasNDD())) {
20865 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20866 }
20867 if ((!Subtarget->hasNDD())) {
20868 return fastEmitInst_ri(MachineInstOpcode: X86::OR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20869 }
20870 return Register();
20871}
20872
20873Register fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20874 if (RetVT.SimpleTy != MVT::i32)
20875 return Register();
20876 if ((Subtarget->hasNDD())) {
20877 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20878 }
20879 if ((!Subtarget->hasNDD())) {
20880 return fastEmitInst_ri(MachineInstOpcode: X86::OR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20881 }
20882 return Register();
20883}
20884
20885Register fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20886 switch (VT.SimpleTy) {
20887 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, imm1);
20888 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, imm1);
20889 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, imm1);
20890 default: return Register();
20891 }
20892}
20893
20894// FastEmit functions for ISD::ROTL.
20895
20896Register fastEmit_ISD_ROTL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20897 if (RetVT.SimpleTy != MVT::i8)
20898 return Register();
20899 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20900 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20901 }
20902 if ((!Subtarget->hasNDD())) {
20903 return fastEmitInst_ri(MachineInstOpcode: X86::ROL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20904 }
20905 return Register();
20906}
20907
20908Register fastEmit_ISD_ROTL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20909 if (RetVT.SimpleTy != MVT::i16)
20910 return Register();
20911 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20912 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20913 }
20914 if ((!Subtarget->hasNDD())) {
20915 return fastEmitInst_ri(MachineInstOpcode: X86::ROL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20916 }
20917 return Register();
20918}
20919
20920Register fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20921 if (RetVT.SimpleTy != MVT::i32)
20922 return Register();
20923 if ((Subtarget->hasFastSHLDRotate())) {
20924 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20925 }
20926 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20927 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20928 }
20929 if ((!Subtarget->hasNDD())) {
20930 return fastEmitInst_ri(MachineInstOpcode: X86::ROL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20931 }
20932 return Register();
20933}
20934
20935Register fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20936 if (RetVT.SimpleTy != MVT::i64)
20937 return Register();
20938 if ((Subtarget->hasFastSHLDRotate())) {
20939 return fastEmitInst_ri(MachineInstOpcode: X86::SHLDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20940 }
20941 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20942 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20943 }
20944 if ((!Subtarget->hasNDD())) {
20945 return fastEmitInst_ri(MachineInstOpcode: X86::ROL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
20946 }
20947 return Register();
20948}
20949
20950Register fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
20951 switch (VT.SimpleTy) {
20952 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_ri(RetVT, Op0, imm1);
20953 case MVT::i16: return fastEmit_ISD_ROTL_MVT_i16_ri(RetVT, Op0, imm1);
20954 case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, imm1);
20955 case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, imm1);
20956 default: return Register();
20957 }
20958}
20959
20960// FastEmit functions for ISD::ROTR.
20961
20962Register fastEmit_ISD_ROTR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20963 if (RetVT.SimpleTy != MVT::i8)
20964 return Register();
20965 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20966 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20967 }
20968 if ((!Subtarget->hasNDD())) {
20969 return fastEmitInst_ri(MachineInstOpcode: X86::ROR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
20970 }
20971 return Register();
20972}
20973
20974Register fastEmit_ISD_ROTR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20975 if (RetVT.SimpleTy != MVT::i16)
20976 return Register();
20977 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20978 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20979 }
20980 if ((!Subtarget->hasNDD())) {
20981 return fastEmitInst_ri(MachineInstOpcode: X86::ROR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
20982 }
20983 return Register();
20984}
20985
20986Register fastEmit_ISD_ROTR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
20987 if (RetVT.SimpleTy != MVT::i32)
20988 return Register();
20989 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
20990 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri_EVEX, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20991 }
20992 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
20993 return fastEmitInst_ri(MachineInstOpcode: X86::RORX32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20994 }
20995 if ((Subtarget->hasFastSHLDRotate())) {
20996 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
20997 }
20998 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
20999 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21000 }
21001 if ((!Subtarget->hasNDD())) {
21002 return fastEmitInst_ri(MachineInstOpcode: X86::ROR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21003 }
21004 return Register();
21005}
21006
21007Register fastEmit_ISD_ROTR_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21008 if (RetVT.SimpleTy != MVT::i64)
21009 return Register();
21010 if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
21011 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri_EVEX, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21012 }
21013 if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
21014 return fastEmitInst_ri(MachineInstOpcode: X86::RORX64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21015 }
21016 if ((Subtarget->hasFastSHLDRotate())) {
21017 return fastEmitInst_ri(MachineInstOpcode: X86::SHRDROT64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21018 }
21019 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21020 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21021 }
21022 if ((!Subtarget->hasNDD())) {
21023 return fastEmitInst_ri(MachineInstOpcode: X86::ROR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21024 }
21025 return Register();
21026}
21027
21028Register fastEmit_ISD_ROTR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21029 switch (VT.SimpleTy) {
21030 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_ri(RetVT, Op0, imm1);
21031 case MVT::i16: return fastEmit_ISD_ROTR_MVT_i16_ri(RetVT, Op0, imm1);
21032 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri(RetVT, Op0, imm1);
21033 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_ri(RetVT, Op0, imm1);
21034 default: return Register();
21035 }
21036}
21037
21038// FastEmit functions for ISD::SHL.
21039
21040Register fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21041 if (RetVT.SimpleTy != MVT::i8)
21042 return Register();
21043 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21044 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21045 }
21046 if ((!Subtarget->hasNDD())) {
21047 return fastEmitInst_ri(MachineInstOpcode: X86::SHL8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21048 }
21049 return Register();
21050}
21051
21052Register fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21053 if (RetVT.SimpleTy != MVT::i16)
21054 return Register();
21055 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21056 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21057 }
21058 if ((!Subtarget->hasNDD())) {
21059 return fastEmitInst_ri(MachineInstOpcode: X86::SHL16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21060 }
21061 return Register();
21062}
21063
21064Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21065 if (RetVT.SimpleTy != MVT::i32)
21066 return Register();
21067 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21068 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21069 }
21070 if ((!Subtarget->hasNDD())) {
21071 return fastEmitInst_ri(MachineInstOpcode: X86::SHL32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21072 }
21073 return Register();
21074}
21075
21076Register fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21077 if (RetVT.SimpleTy != MVT::i64)
21078 return Register();
21079 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21080 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21081 }
21082 if ((!Subtarget->hasNDD())) {
21083 return fastEmitInst_ri(MachineInstOpcode: X86::SHL64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21084 }
21085 return Register();
21086}
21087
21088Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21089 switch (VT.SimpleTy) {
21090 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, imm1);
21091 case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, imm1);
21092 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
21093 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, imm1);
21094 default: return Register();
21095 }
21096}
21097
21098// FastEmit functions for ISD::SRA.
21099
21100Register fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21101 if (RetVT.SimpleTy != MVT::i8)
21102 return Register();
21103 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21104 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21105 }
21106 if ((!Subtarget->hasNDD())) {
21107 return fastEmitInst_ri(MachineInstOpcode: X86::SAR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21108 }
21109 return Register();
21110}
21111
21112Register fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21113 if (RetVT.SimpleTy != MVT::i16)
21114 return Register();
21115 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21116 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21117 }
21118 if ((!Subtarget->hasNDD())) {
21119 return fastEmitInst_ri(MachineInstOpcode: X86::SAR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21120 }
21121 return Register();
21122}
21123
21124Register fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21125 if (RetVT.SimpleTy != MVT::i32)
21126 return Register();
21127 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21128 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21129 }
21130 if ((!Subtarget->hasNDD())) {
21131 return fastEmitInst_ri(MachineInstOpcode: X86::SAR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21132 }
21133 return Register();
21134}
21135
21136Register fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21137 if (RetVT.SimpleTy != MVT::i64)
21138 return Register();
21139 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21140 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21141 }
21142 if ((!Subtarget->hasNDD())) {
21143 return fastEmitInst_ri(MachineInstOpcode: X86::SAR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21144 }
21145 return Register();
21146}
21147
21148Register fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21149 switch (VT.SimpleTy) {
21150 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, imm1);
21151 case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, imm1);
21152 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, imm1);
21153 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, imm1);
21154 default: return Register();
21155 }
21156}
21157
21158// FastEmit functions for ISD::SRL.
21159
21160Register fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21161 if (RetVT.SimpleTy != MVT::i8)
21162 return Register();
21163 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21164 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21165 }
21166 if ((!Subtarget->hasNDD())) {
21167 return fastEmitInst_ri(MachineInstOpcode: X86::SHR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21168 }
21169 return Register();
21170}
21171
21172Register fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21173 if (RetVT.SimpleTy != MVT::i16)
21174 return Register();
21175 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21176 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21177 }
21178 if ((!Subtarget->hasNDD())) {
21179 return fastEmitInst_ri(MachineInstOpcode: X86::SHR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21180 }
21181 return Register();
21182}
21183
21184Register fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21185 if (RetVT.SimpleTy != MVT::i32)
21186 return Register();
21187 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21188 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21189 }
21190 if ((!Subtarget->hasNDD())) {
21191 return fastEmitInst_ri(MachineInstOpcode: X86::SHR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21192 }
21193 return Register();
21194}
21195
21196Register fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21197 if (RetVT.SimpleTy != MVT::i64)
21198 return Register();
21199 if ((Subtarget->hasNDD()) && (Subtarget->is64Bit())) {
21200 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21201 }
21202 if ((!Subtarget->hasNDD())) {
21203 return fastEmitInst_ri(MachineInstOpcode: X86::SHR64ri, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21204 }
21205 return Register();
21206}
21207
21208Register fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21209 switch (VT.SimpleTy) {
21210 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, imm1);
21211 case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, imm1);
21212 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, imm1);
21213 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, imm1);
21214 default: return Register();
21215 }
21216}
21217
21218// FastEmit functions for ISD::SUB.
21219
21220Register fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21221 if (RetVT.SimpleTy != MVT::i8)
21222 return Register();
21223 if ((Subtarget->hasNDD())) {
21224 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21225 }
21226 if ((!Subtarget->hasNDD())) {
21227 return fastEmitInst_ri(MachineInstOpcode: X86::SUB8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21228 }
21229 return Register();
21230}
21231
21232Register fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21233 if (RetVT.SimpleTy != MVT::i16)
21234 return Register();
21235 if ((Subtarget->hasNDD())) {
21236 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21237 }
21238 if ((!Subtarget->hasNDD())) {
21239 return fastEmitInst_ri(MachineInstOpcode: X86::SUB16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21240 }
21241 return Register();
21242}
21243
21244Register fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21245 if (RetVT.SimpleTy != MVT::i32)
21246 return Register();
21247 if ((Subtarget->hasNDD())) {
21248 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21249 }
21250 if ((!Subtarget->hasNDD())) {
21251 return fastEmitInst_ri(MachineInstOpcode: X86::SUB32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21252 }
21253 return Register();
21254}
21255
21256Register fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21257 switch (VT.SimpleTy) {
21258 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, imm1);
21259 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, imm1);
21260 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, imm1);
21261 default: return Register();
21262 }
21263}
21264
21265// FastEmit functions for ISD::XOR.
21266
21267Register fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21268 if (RetVT.SimpleTy != MVT::i8)
21269 return Register();
21270 if ((Subtarget->hasNDD())) {
21271 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri_ND, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21272 }
21273 if ((!Subtarget->hasNDD())) {
21274 return fastEmitInst_ri(MachineInstOpcode: X86::XOR8ri, RC: &X86::GR8RegClass, Op0, Imm: imm1);
21275 }
21276 return Register();
21277}
21278
21279Register fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21280 if (RetVT.SimpleTy != MVT::i16)
21281 return Register();
21282 if ((Subtarget->hasNDD())) {
21283 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri_ND, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21284 }
21285 if ((!Subtarget->hasNDD())) {
21286 return fastEmitInst_ri(MachineInstOpcode: X86::XOR16ri, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21287 }
21288 return Register();
21289}
21290
21291Register fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21292 if (RetVT.SimpleTy != MVT::i32)
21293 return Register();
21294 if ((Subtarget->hasNDD())) {
21295 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri_ND, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21296 }
21297 if ((!Subtarget->hasNDD())) {
21298 return fastEmitInst_ri(MachineInstOpcode: X86::XOR32ri, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21299 }
21300 return Register();
21301}
21302
21303Register fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21304 switch (VT.SimpleTy) {
21305 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, imm1);
21306 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, imm1);
21307 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, imm1);
21308 default: return Register();
21309 }
21310}
21311
21312// FastEmit functions for X86ISD::BT.
21313
21314Register fastEmit_X86ISD_BT_MVT_i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21315 if (RetVT.SimpleTy != MVT::i32)
21316 return Register();
21317 return fastEmitInst_ri(MachineInstOpcode: X86::BT16ri8, RC: &X86::GR16RegClass, Op0, Imm: imm1);
21318}
21319
21320Register fastEmit_X86ISD_BT_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21321 if (RetVT.SimpleTy != MVT::i32)
21322 return Register();
21323 return fastEmitInst_ri(MachineInstOpcode: X86::BT32ri8, RC: &X86::GR32RegClass, Op0, Imm: imm1);
21324}
21325
21326Register fastEmit_X86ISD_BT_MVT_i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
21327 if (RetVT.SimpleTy != MVT::i32)
21328 return Register();
21329 return fastEmitInst_ri(MachineInstOpcode: X86::BT64ri8, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21330}
21331
21332Register fastEmit_X86ISD_BT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21333 switch (VT.SimpleTy) {
21334 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_ri(RetVT, Op0, imm1);
21335 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_ri(RetVT, Op0, imm1);
21336 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_ri(RetVT, Op0, imm1);
21337 default: return Register();
21338 }
21339}
21340
21341// Top-level FastEmit function.
21342
21343Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
21344 if (VT == MVT::i64 && Predicate_i64immSExt32(Imm: imm1))
21345 if (Register Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, imm1))
21346 return Reg;
21347
21348 switch (Opcode) {
21349 case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, imm1);
21350 case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, imm1);
21351 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
21352 case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, imm1);
21353 case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, imm1);
21354 case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, imm1);
21355 case ISD::ROTR: return fastEmit_ISD_ROTR_ri(VT, RetVT, Op0, imm1);
21356 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
21357 case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, imm1);
21358 case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, imm1);
21359 case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, imm1);
21360 case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, imm1);
21361 case X86ISD::BT: return fastEmit_X86ISD_BT_ri(VT, RetVT, Op0, imm1);
21362 default: return Register();
21363 }
21364}
21365
21366// FastEmit functions for ISD::ADD.
21367
21368Register fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21369 if (RetVT.SimpleTy != MVT::i64)
21370 return Register();
21371 if ((Subtarget->hasNDD())) {
21372 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21373 }
21374 if ((!Subtarget->hasNDD())) {
21375 return fastEmitInst_ri(MachineInstOpcode: X86::ADD64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21376 }
21377 return Register();
21378}
21379
21380Register fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21381 switch (VT.SimpleTy) {
21382 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21383 default: return Register();
21384 }
21385}
21386
21387// FastEmit functions for ISD::AND.
21388
21389Register fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21390 if (RetVT.SimpleTy != MVT::i64)
21391 return Register();
21392 if ((Subtarget->hasNDD())) {
21393 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21394 }
21395 if ((!Subtarget->hasNDD())) {
21396 return fastEmitInst_ri(MachineInstOpcode: X86::AND64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21397 }
21398 return Register();
21399}
21400
21401Register fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21402 switch (VT.SimpleTy) {
21403 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21404 default: return Register();
21405 }
21406}
21407
21408// FastEmit functions for ISD::MUL.
21409
21410Register fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21411 if (RetVT.SimpleTy != MVT::i64)
21412 return Register();
21413 return fastEmitInst_ri(MachineInstOpcode: X86::IMUL64rri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21414}
21415
21416Register fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21417 switch (VT.SimpleTy) {
21418 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21419 default: return Register();
21420 }
21421}
21422
21423// FastEmit functions for ISD::OR.
21424
21425Register fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21426 if (RetVT.SimpleTy != MVT::i64)
21427 return Register();
21428 if ((Subtarget->hasNDD())) {
21429 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21430 }
21431 if ((!Subtarget->hasNDD())) {
21432 return fastEmitInst_ri(MachineInstOpcode: X86::OR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21433 }
21434 return Register();
21435}
21436
21437Register fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21438 switch (VT.SimpleTy) {
21439 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21440 default: return Register();
21441 }
21442}
21443
21444// FastEmit functions for ISD::SUB.
21445
21446Register fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21447 if (RetVT.SimpleTy != MVT::i64)
21448 return Register();
21449 if ((Subtarget->hasNDD())) {
21450 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21451 }
21452 if ((!Subtarget->hasNDD())) {
21453 return fastEmitInst_ri(MachineInstOpcode: X86::SUB64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21454 }
21455 return Register();
21456}
21457
21458Register fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21459 switch (VT.SimpleTy) {
21460 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21461 default: return Register();
21462 }
21463}
21464
21465// FastEmit functions for ISD::XOR.
21466
21467Register fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, Register Op0, uint64_t imm1) {
21468 if (RetVT.SimpleTy != MVT::i64)
21469 return Register();
21470 if ((Subtarget->hasNDD())) {
21471 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32_ND, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21472 }
21473 if ((!Subtarget->hasNDD())) {
21474 return fastEmitInst_ri(MachineInstOpcode: X86::XOR64ri32, RC: &X86::GR64RegClass, Op0, Imm: imm1);
21475 }
21476 return Register();
21477}
21478
21479Register fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
21480 switch (VT.SimpleTy) {
21481 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
21482 default: return Register();
21483 }
21484}
21485
21486// Top-level FastEmit function.
21487
21488Register fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
21489 switch (Opcode) {
21490 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21491 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21492 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21493 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21494 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21495 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
21496 default: return Register();
21497 }
21498}
21499
21500// FastEmit functions for ISD::Constant.
21501
21502Register fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) {
21503 if (RetVT.SimpleTy != MVT::i8)
21504 return Register();
21505 return fastEmitInst_i(MachineInstOpcode: X86::MOV8ri, RC: &X86::GR8RegClass, Imm: imm0);
21506}
21507
21508Register fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) {
21509 if (RetVT.SimpleTy != MVT::i16)
21510 return Register();
21511 return fastEmitInst_i(MachineInstOpcode: X86::MOV16ri, RC: &X86::GR16RegClass, Imm: imm0);
21512}
21513
21514Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
21515 if (RetVT.SimpleTy != MVT::i32)
21516 return Register();
21517 return fastEmitInst_i(MachineInstOpcode: X86::MOV32ri, RC: &X86::GR32RegClass, Imm: imm0);
21518}
21519
21520Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
21521 if (RetVT.SimpleTy != MVT::i64)
21522 return Register();
21523 return fastEmitInst_i(MachineInstOpcode: X86::MOV64ri, RC: &X86::GR64RegClass, Imm: imm0);
21524}
21525
21526Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
21527 switch (VT.SimpleTy) {
21528 case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0);
21529 case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0);
21530 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
21531 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
21532 default: return Register();
21533 }
21534}
21535
21536// Top-level FastEmit function.
21537
21538Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
21539 switch (Opcode) {
21540 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
21541 default: return Register();
21542 }
21543}
21544
21545