1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit x86/x86_64 Builtin calls as CIR or a function
10// call to be later resolved.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CIRGenBuilder.h"
15#include "CIRGenFunction.h"
16#include "CIRGenModule.h"
17#include "mlir/IR/Attributes.h"
18#include "mlir/IR/BuiltinAttributes.h"
19#include "mlir/IR/Location.h"
20#include "mlir/IR/Types.h"
21#include "mlir/IR/ValueRange.h"
22#include "clang/Basic/Builtins.h"
23#include "clang/Basic/TargetBuiltins.h"
24#include "clang/CIR/Dialect/IR/CIRAttrs.h"
25#include "clang/CIR/Dialect/IR/CIRTypes.h"
26#include "clang/CIR/MissingFeatures.h"
27#include "clang/CodeGenUtils/TargetUtils.h"
28#include "llvm/ADT/Sequence.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/TargetParser/X86TargetParser.h"
31#include <string>
32
33using namespace clang;
34using namespace clang::CIRGen;
35
36static mlir::Value emitVectorFCmp(CIRGenFunction &cgf, const CallExpr &expr,
37 llvm::SmallVector<mlir::Value> &ops,
38 cir::FCmpOpKind pred, bool signaling) {
39 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, &expr);
40 clang::CIRGen::CIRGenBuilderTy &builder = cgf.getBuilder();
41 mlir::Value cmp = builder.createVecFCompare(loc: cgf.getLoc(srcLoc: expr.getExprLoc()),
42 kind: pred, lhs: ops[0], rhs: ops[1], signaling);
43 return builder.createBitcast(src: cmp, newTy: ops[0].getType());
44}
45
46struct X86FCmpInfo {
47 cir::FCmpOpKind pred;
48 bool isSignaling;
49};
50
51/// Decodes comparison condition of x86 cmppps/cmppd* builtins which is an
52/// integer in the range [0, 31]. Predicates for 16-31 repeat the 0-15
53/// predicates and differ only in signaling behavior.
54static X86FCmpInfo getX86FCmpInfo(unsigned cc) {
55 bool inverseSig = cc & 0x10;
56 const auto createVal = [inverseSig](cir::FCmpOpKind kind, bool isSignaling) {
57 return X86FCmpInfo{.pred: kind, .isSignaling: static_cast<bool>(inverseSig ^ isSignaling)};
58 };
59 switch (cc & 0xf) {
60 case 0x00:
61 return createVal(cir::FCmpOpKind::oeq, /*isSignaling=*/false);
62 case 0x01:
63 return createVal(cir::FCmpOpKind::olt, /*isSignaling=*/true);
64 case 0x02:
65 return createVal(cir::FCmpOpKind::ole, /*isSignaling=*/true);
66 case 0x03:
67 return createVal(cir::FCmpOpKind::uno, /*isSignaling=*/false);
68 case 0x04:
69 return createVal(cir::FCmpOpKind::une, /*isSignaling=*/false);
70 case 0x05:
71 return createVal(cir::FCmpOpKind::uge, /*isSignaling=*/true);
72 case 0x06:
73 return createVal(cir::FCmpOpKind::ugt, /*isSignaling=*/true);
74 case 0x07:
75 return createVal(cir::FCmpOpKind::ord, /*isSignaling=*/false);
76 case 0x08:
77 return createVal(cir::FCmpOpKind::ueq, /*isSignaling=*/false);
78 case 0x09:
79 return createVal(cir::FCmpOpKind::ult, /*isSignaling=*/true);
80 case 0x0a:
81 return createVal(cir::FCmpOpKind::ule, /*isSignaling=*/true);
82 case 0x0b:
83 return createVal(cir::FCmpOpKind::_false, /*isSignaling=*/false);
84 case 0x0c:
85 return createVal(cir::FCmpOpKind::one, /*isSignaling=*/false);
86 case 0x0d:
87 return createVal(cir::FCmpOpKind::oge, /*isSignaling=*/true);
88 case 0x0e:
89 return createVal(cir::FCmpOpKind::ogt, /*isSignaling=*/true);
90 case 0x0f:
91 return createVal(cir::FCmpOpKind::_true, /*isSignaling=*/false);
92 default:
93 llvm_unreachable("Unhandled cc");
94 }
95}
96
97/// Returns the legacy x86 intrinsic that implements the given x86 cmppps/cmppd*
98/// builtin.
99static llvm::StringRef getX86FCmpIntrinsicName(unsigned builtinID) {
100 switch (builtinID) {
101 default:
102 llvm_unreachable("Unexpected builtin");
103 case X86::BI__builtin_ia32_cmpps:
104 return "x86.sse.cmp.ps";
105 case X86::BI__builtin_ia32_cmpps256:
106 return "x86.avx.cmp.ps.256";
107 case X86::BI__builtin_ia32_cmppd:
108 return "x86.sse2.cmp.pd";
109 case X86::BI__builtin_ia32_cmppd256:
110 return "x86.avx.cmp.pd.256";
111 case X86::BI__builtin_ia32_cmpph128_mask:
112 return "x86.avx512fp16.mask.cmp.ph.128";
113 case X86::BI__builtin_ia32_cmpph256_mask:
114 return "x86.avx512fp16.mask.cmp.ph.256";
115 case X86::BI__builtin_ia32_cmpph512_mask:
116 return "x86.avx512fp16.mask.cmp.ph.512";
117 case X86::BI__builtin_ia32_cmpps512_mask:
118 return "x86.avx512.mask.cmp.ps.512";
119 case X86::BI__builtin_ia32_cmppd512_mask:
120 return "x86.avx512.mask.cmp.pd.512";
121 case X86::BI__builtin_ia32_cmpps128_mask:
122 return "x86.avx512.mask.cmp.ps.128";
123 case X86::BI__builtin_ia32_cmpps256_mask:
124 return "x86.avx512.mask.cmp.ps.256";
125 case X86::BI__builtin_ia32_cmppd128_mask:
126 return "x86.avx512.mask.cmp.pd.128";
127 case X86::BI__builtin_ia32_cmppd256_mask:
128 return "x86.avx512.mask.cmp.pd.256";
129 }
130}
131
132static mlir::Value getMaskVecValue(CIRGenBuilderTy &builder, mlir::Location loc,
133 mlir::Value mask, unsigned numElems) {
134 auto maskTy = cir::VectorType::getFixed(
135 elementType: builder.getSIntNTy(n: 1), size: cast<cir::IntType>(Val: mask.getType()).getWidth());
136 mlir::Value maskVec = builder.createBitcast(src: mask, newTy: maskTy);
137
138 // If we have less than 8 elements, then the starting mask was an i8 and
139 // we need to extract down to the right number of elements.
140 if (numElems < 8) {
141 SmallVector<mlir::Attribute, 4> indices;
142 mlir::Type i32Ty = builder.getSInt32Ty();
143 for (auto i : llvm::seq<unsigned>(Begin: 0, End: numElems))
144 indices.push_back(Elt: cir::IntAttr::get(type: i32Ty, value: i));
145
146 maskVec = builder.createVecShuffle(loc, vec1: maskVec, vec2: maskVec, maskAttrs: indices);
147 }
148 return maskVec;
149}
150
151static mlir::Value emitX86Round(CIRGenFunction &cgf, unsigned builtinID,
152 const CallExpr *expr,
153 SmallVector<mlir::Value> &ops) {
154 bool isScalar = builtinID == X86::BI__builtin_ia32_roundss ||
155 builtinID == X86::BI__builtin_ia32_roundsd;
156
157 unsigned roundingControl =
158 cgf.getZExtIntValueFromConstOp(val: isScalar ? ops[2] : ops[1]);
159
160 constexpr unsigned mxcsrMask = 0b100;
161 constexpr unsigned fRoundNoExcMask = 0b1000;
162 unsigned useMXCSR = mxcsrMask & roundingControl;
163 unsigned fRoundNoExc = fRoundNoExcMask & roundingControl;
164
165 CIRGenBuilderTy &builder = cgf.getBuilder();
166 mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc());
167
168 if (useMXCSR || !fRoundNoExc) {
169 StringRef intrinsicName;
170 switch (builtinID) {
171 default:
172 llvm_unreachable("Unexpected builtin");
173 case X86::BI__builtin_ia32_roundps:
174 intrinsicName = "x86.sse41.round.ps";
175 break;
176 case X86::BI__builtin_ia32_roundpd:
177 intrinsicName = "x86.sse41.round.pd";
178 break;
179 case X86::BI__builtin_ia32_roundps256:
180 intrinsicName = "x86.avx.round.ps.256";
181 break;
182 case X86::BI__builtin_ia32_roundpd256:
183 intrinsicName = "x86.avx.round.pd.256";
184 break;
185 case X86::BI__builtin_ia32_roundss:
186 intrinsicName = "x86.sse41.round.ss";
187 break;
188 case X86::BI__builtin_ia32_roundsd:
189 intrinsicName = "x86.sse41.round.sd";
190 break;
191 }
192
193 mlir::Type resTy = ops[0].getType();
194 return builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy, op&: ops);
195 }
196
197 auto emitRoundOp = [&](mlir::Value x) -> mlir::Value {
198 constexpr unsigned roundingMask = 0b11;
199 unsigned roundingMode = roundingControl & roundingMask;
200
201 if (builder.getIsFPConstrained()) {
202 CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, expr);
203
204 builder.setDefaultConstrainedExcept(clang::LangOptions::FPE_Ignore);
205 cir::FenvAttr fenv = builder.getConstrainedFPAttr();
206
207 switch (roundingMode) {
208 default:
209 llvm_unreachable("Invalid rounding mode");
210 case 0b00:
211 return cir::RoundEvenOp::create(builder, location: loc, src: x, fenv);
212 case 0b01:
213 return cir::FloorOp::create(builder, location: loc, src: x, fenv);
214 case 0b10:
215 return cir::CeilOp::create(builder, location: loc, src: x, fenv);
216 case 0b11:
217 return cir::TruncOp::create(builder, location: loc, src: x, fenv);
218 }
219 }
220
221 switch (roundingMode) {
222 default:
223 llvm_unreachable("Invalid rounding mode");
224 case 0b00:
225 return cir::RoundEvenOp::create(builder, location: loc, src: x);
226 case 0b01:
227 return cir::FloorOp::create(builder, location: loc, src: x);
228 case 0b10:
229 return cir::CeilOp::create(builder, location: loc, src: x);
230 case 0b11:
231 return cir::TruncOp::create(builder, location: loc, src: x);
232 }
233 };
234
235 if (isScalar) {
236 mlir::Value valAt0 = builder.createExtractElement(loc, vec: ops[1], idx: 0);
237 mlir::Value roundedAt0 = emitRoundOp(valAt0);
238 return builder.createInsertElement(loc, vec: ops[0], newElt: roundedAt0, idx: 0);
239 }
240
241 return emitRoundOp(ops[0]);
242}
243
244static mlir::Value emitX86CompressStore(CIRGenBuilderTy &builder,
245 mlir::Location loc,
246 ArrayRef<mlir::Value> ops) {
247 auto resultTy = cast<cir::VectorType>(Val: ops[1].getType());
248 mlir::Value maskValue =
249 getMaskVecValue(builder, loc, mask: ops[2], numElems: resultTy.getFixedSize());
250
251 return builder.emitIntrinsicCallOp(
252 loc, str: "masked.compressstore", resTy: cir::VoidType::get(ctx: builder.getContext()),
253 op: mlir::ValueRange{ops[1], ops[0], maskValue});
254}
255
256// Builds the VecShuffleOp for pshuflw and pshufhw x86 builtins.
257//
258// The vector is split into lanes of 8 word elements (16 bits). The lower or
259// upper half of each lane, controlled by `isLow`, is shuffled in the following
260// way: The immediate is truncated to 8 bits, separated into 4 2-bit fields. The
261// i-th field's value represents the resulting index of the i-th element in the
262// half lane after shuffling. The other half of the lane remains unchanged.
263static cir::VecShuffleOp emitPshufWord(CIRGenBuilderTy &builder,
264 const mlir::Value vec,
265 const mlir::Value immediate,
266 const mlir::Location loc,
267 const bool isLow) {
268 uint32_t imm = CIRGenFunction::getZExtIntValueFromConstOp(val: immediate);
269
270 auto vecTy = cast<cir::VectorType>(Val: vec.getType());
271 unsigned numElts = vecTy.getFixedSize();
272
273 unsigned firstHalfStart = isLow ? 0 : 4;
274 unsigned secondHalfStart = 4 - firstHalfStart;
275
276 // Splat the 8-bits of immediate 4 times to help the loop wrap around.
277 imm = (imm & 0xff) * 0x01010101;
278
279 int64_t indices[32];
280 for (unsigned l = 0; l != numElts; l += 8) {
281 for (unsigned i = firstHalfStart; i != firstHalfStart + 4; ++i) {
282 indices[l + i] = l + (imm & 3) + firstHalfStart;
283 imm >>= 2;
284 }
285 for (unsigned i = secondHalfStart; i != secondHalfStart + 4; ++i)
286 indices[l + i] = l + i;
287 }
288
289 return builder.createVecShuffle(loc, vec1: vec, mask: ArrayRef(indices, numElts));
290}
291
292// Builds the shuffle mask for pshufd and shufpd/shufps x86 builtins.
293// The shuffle mask is written to outIndices.
294static void
295computeFullLaneShuffleMask(CIRGenFunction &cgf, const mlir::Value vec,
296 uint32_t imm, const bool isShufP,
297 llvm::SmallVectorImpl<int64_t> &outIndices) {
298 auto vecTy = cast<cir::VectorType>(Val: vec.getType());
299 unsigned numElts = vecTy.getFixedSize();
300 unsigned numLanes = cgf.cgm.getDataLayout().getTypeSizeInBits(ty: vecTy) / 128;
301 unsigned numLaneElts = numElts / numLanes;
302
303 // Splat the 8-bits of immediate 4 times to help the loop wrap around.
304 imm = (imm & 0xff) * 0x01010101;
305
306 for (unsigned l = 0; l != numElts; l += numLaneElts) {
307 for (unsigned i = 0; i != numLaneElts; ++i) {
308 uint32_t idx = imm % numLaneElts;
309 imm /= numLaneElts;
310 if (isShufP && i >= (numLaneElts / 2))
311 idx += numElts;
312 outIndices[l + i] = l + idx;
313 }
314 }
315
316 outIndices.resize(N: numElts);
317}
318
319static mlir::Value emitPrefetch(CIRGenFunction &cgf, unsigned builtinID,
320 const CallExpr *e,
321 const SmallVector<mlir::Value> &ops) {
322 CIRGenBuilderTy &builder = cgf.getBuilder();
323 mlir::Location location = cgf.getLoc(srcLoc: e->getExprLoc());
324 mlir::Type voidTy = builder.getVoidTy();
325 mlir::Value address = builder.createPtrBitcast(src: ops[0], newPointeeTy: voidTy);
326 bool isWrite{};
327 int locality{};
328
329 assert(builtinID == X86::BI_mm_prefetch || builtinID == X86::BI_m_prefetchw ||
330 builtinID == X86::BI_m_prefetch && "Expected prefetch builtin");
331
332 if (builtinID == X86::BI_mm_prefetch) {
333 int hint = cgf.getSExtIntValueFromConstOp(val: ops[1]);
334 isWrite = (hint >> 2) & 0x1;
335 locality = hint & 0x3;
336 } else {
337 isWrite = (builtinID == X86::BI_m_prefetchw);
338 locality = 0x3;
339 }
340
341 cir::PrefetchOp::create(builder, location, addr: address, locality, isWrite);
342 return {};
343}
344
345static mlir::Value emitX86CompressExpand(CIRGenBuilderTy &builder,
346 mlir::Location loc, mlir::Value source,
347 mlir::Value mask,
348 mlir::Value inputVector,
349 const std::string &id) {
350 auto resultTy = cast<cir::VectorType>(Val: mask.getType());
351 mlir::Value maskValue =
352 getMaskVecValue(builder, loc, mask: inputVector,
353 numElems: cast<cir::VectorType>(Val&: resultTy).getFixedSize());
354 return builder.emitIntrinsicCallOp(loc, str: id, resTy: resultTy,
355 op: mlir::ValueRange{source, mask, maskValue});
356}
357
358static mlir::Value
359emitEncodeKey(mlir::MLIRContext *context, CIRGenBuilderTy &builder,
360 const mlir::Location &location, mlir::ValueRange inputOperands,
361 mlir::Value outputOperand, std::uint8_t vecOutputCount,
362 const std::string &intrinsicName, std::uint8_t numResults) {
363 cir::VectorType resVector =
364 cir::VectorType::getFixed(elementType: builder.getUInt64Ty(), size: 2);
365 llvm::SmallVector<mlir::Type> members{builder.getUInt32Ty()};
366 llvm::append_range(C&: members,
367 R: llvm::SmallVector<mlir::Type>(vecOutputCount, resVector));
368 cir::StructType resRecord = cir::StructType::get(
369 context, members, /*packed=*/false,
370 /*is_class=*/false, member_kinds: cir::RecordType::getAllDataKinds(members));
371
372 mlir::Value outputPtr =
373 builder.createBitcast(src: outputOperand, newTy: cir::PointerType::get(pointee: resVector));
374 mlir::Value call = builder.emitIntrinsicCallOp(loc: location, str: intrinsicName,
375 resTy: resRecord, op&: inputOperands);
376 for (std::uint8_t i = 0; i < numResults; ++i) {
377 mlir::Value vecValue =
378 cir::ExtractMemberOp::create(builder, location, record: call, index: i + 1);
379 mlir::Value index = builder.getSInt32(c: i, loc: location);
380 mlir::Value ptr = builder.createPtrStride(loc: location, base: outputPtr, stride: index);
381 builder.createStore(loc: location, val: vecValue, dst: Address{ptr, CharUnits::One()});
382 }
383 return cir::ExtractMemberOp::create(builder, location, record: call, index: 0);
384}
385
386static mlir::Value emitX86Select(CIRGenBuilderTy &builder, mlir::Location loc,
387 mlir::Value mask, mlir::Value op0,
388 mlir::Value op1) {
389 auto constOp = mlir::dyn_cast_or_null<cir::ConstantOp>(Val: mask.getDefiningOp());
390 // If the mask is all ones just return first argument.
391 if (constOp && constOp.isAllOnesValue())
392 return op0;
393
394 mask = getMaskVecValue(builder, loc, mask,
395 numElems: cast<cir::VectorType>(Val: op0.getType()).getFixedSize());
396
397 return cir::VecTernaryOp::create(builder, location: loc, cond: mask, lhs: op0, rhs: op1);
398}
399
400// Helper function to extract zero-bit from a mask as a boolean
401static mlir::Value getMaskZeroBitAsBool(CIRGenBuilderTy &builder,
402 mlir::Location loc, mlir::Value mask) {
403 // Get the mask as a vector of i1 and extract bit 0
404 auto intTy = mlir::dyn_cast<cir::IntType>(Val: mask.getType());
405 assert(intTy && "mask must be an integer type");
406 unsigned width = intTy.getWidth();
407
408 auto maskVecTy = cir::VectorType::getFixed(elementType: builder.getSIntNTy(n: 1), size: width);
409 mlir::Value maskVec = builder.createBitcast(src: mask, newTy: maskVecTy);
410
411 // Extract bit 0 from the mask vector
412 mlir::Value bit0 = builder.createExtractElement(loc, vec: maskVec, idx: uint64_t(0));
413
414 // Convert i1 to bool for select
415 auto boolTy = cir::BoolType::get(ctx: builder.getContext());
416 return cir::CastOp::create(builder, location: loc, result: boolTy, kind: cir::CastKind::int_to_bool,
417 src: bit0);
418}
419
420static mlir::Value emitX86ScalarSelect(CIRGenBuilderTy &builder,
421 mlir::Location loc, mlir::Value mask,
422 mlir::Value op0, mlir::Value op1) {
423
424 // If the mask is all ones just return first argument.
425 if (auto c = mlir::dyn_cast_or_null<cir::ConstantOp>(Val: mask.getDefiningOp()))
426 if (c.isAllOnesValue())
427 return op0;
428
429 mlir::Value cond = getMaskZeroBitAsBool(builder, loc, mask);
430 return builder.createSelect(loc, condition: cond, trueValue: op0, falseValue: op1);
431}
432
433static mlir::Value emitX86MaskAddLogic(CIRGenBuilderTy &builder,
434 mlir::Location loc,
435 const std::string &intrinsicName,
436 SmallVectorImpl<mlir::Value> &ops) {
437
438 auto intTy = cast<cir::IntType>(Val: ops[0].getType());
439 unsigned numElts = intTy.getWidth();
440 mlir::Value lhsVec = getMaskVecValue(builder, loc, mask: ops[0], numElems: numElts);
441 mlir::Value rhsVec = getMaskVecValue(builder, loc, mask: ops[1], numElems: numElts);
442 mlir::Type vecTy = lhsVec.getType();
443 mlir::Value resVec = builder.emitIntrinsicCallOp(
444 loc, str: intrinsicName, resTy: vecTy, op: mlir::ValueRange{lhsVec, rhsVec});
445 return builder.createBitcast(src: resVec, newTy: ops[0].getType());
446}
447
448static mlir::Value emitX86MaskUnpack(CIRGenBuilderTy &builder,
449 mlir::Location loc,
450 const std::string &intrinsicName,
451 SmallVectorImpl<mlir::Value> &ops) {
452 unsigned numElems = cast<cir::IntType>(Val: ops[0].getType()).getWidth();
453
454 // Convert both operands to mask vectors.
455 mlir::Value lhs = getMaskVecValue(builder, loc, mask: ops[0], numElems);
456 mlir::Value rhs = getMaskVecValue(builder, loc, mask: ops[1], numElems);
457
458 mlir::Type i32Ty = builder.getSInt32Ty();
459
460 // Create indices for extracting the first half of each vector.
461 SmallVector<mlir::Attribute, 32> halfIndices;
462 for (auto i : llvm::seq<unsigned>(Begin: 0, End: numElems / 2))
463 halfIndices.push_back(Elt: cir::IntAttr::get(type: i32Ty, value: i));
464
465 // Extract first half of each vector. This gives better codegen than
466 // doing it in a single shuffle.
467 mlir::Value lhsHalf = builder.createVecShuffle(loc, vec1: lhs, vec2: lhs, maskAttrs: halfIndices);
468 mlir::Value rhsHalf = builder.createVecShuffle(loc, vec1: rhs, vec2: rhs, maskAttrs: halfIndices);
469
470 // Create indices for concatenating the vectors.
471 // NOTE: Operands are swapped to match the intrinsic definition.
472 // After the half extraction, both vectors have numElems/2 elements.
473 // In createVecShuffle(rhsHalf, lhsHalf, indices), indices [0..numElems/2-1]
474 // select from rhsHalf, and indices [numElems/2..numElems-1] select from
475 // lhsHalf.
476 SmallVector<mlir::Attribute, 64> concatIndices;
477 for (auto i : llvm::seq<unsigned>(Begin: 0, End: numElems))
478 concatIndices.push_back(Elt: cir::IntAttr::get(type: i32Ty, value: i));
479
480 // Concat the vectors (RHS first, then LHS).
481 mlir::Value res =
482 builder.createVecShuffle(loc, vec1: rhsHalf, vec2: lhsHalf, maskAttrs: concatIndices);
483 return builder.createBitcast(src: res, newTy: ops[0].getType());
484}
485
486template <typename BinOp>
487static mlir::Value
488emitX86MaskLogic(CIRGenBuilderTy &builder, mlir::Location loc,
489 SmallVectorImpl<mlir::Value> &ops, bool invertLHS = false) {
490 unsigned numElts = cast<cir::IntType>(Val: ops[0].getType()).getWidth();
491 mlir::Value lhs = getMaskVecValue(builder, loc, mask: ops[0], numElems: numElts);
492 mlir::Value rhs = getMaskVecValue(builder, loc, mask: ops[1], numElems: numElts);
493
494 if (invertLHS)
495 lhs = builder.createNot(value: lhs);
496 return builder.createBitcast(BinOp::create(builder, loc, lhs, rhs),
497 ops[0].getType());
498}
499
500static mlir::Value emitX86MaskTest(CIRGenBuilderTy &builder, mlir::Location loc,
501 const std::string &intrinsicName,
502 SmallVectorImpl<mlir::Value> &ops) {
503 auto intTy = cast<cir::IntType>(Val: ops[0].getType());
504 unsigned numElts = intTy.getWidth();
505 mlir::Value lhsVec = getMaskVecValue(builder, loc, mask: ops[0], numElems: numElts);
506 mlir::Value rhsVec = getMaskVecValue(builder, loc, mask: ops[1], numElems: numElts);
507 mlir::Type resTy = builder.getSInt32Ty();
508 return builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy,
509 op: mlir::ValueRange{lhsVec, rhsVec});
510}
511
512static mlir::Value emitX86MaskedCompareResult(CIRGenBuilderTy &builder,
513 mlir::Value cmp, unsigned numElts,
514 mlir::Value maskIn,
515 mlir::Location loc) {
516 if (maskIn) {
517 auto c = mlir::dyn_cast_or_null<cir::ConstantOp>(Val: maskIn.getDefiningOp());
518 if (!c || !c.isAllOnesValue())
519 cmp = builder.createAnd(loc, lhs: cmp,
520 rhs: getMaskVecValue(builder, loc, mask: maskIn, numElems: numElts));
521 }
522 if (numElts < 8) {
523 llvm::SmallVector<mlir::Attribute> indices;
524 mlir::Type i64Ty = builder.getSInt64Ty();
525
526 for (unsigned i = 0; i != numElts; ++i)
527 indices.push_back(Elt: cir::IntAttr::get(type: i64Ty, value: i));
528 for (unsigned i = numElts; i != 8; ++i)
529 indices.push_back(Elt: cir::IntAttr::get(type: i64Ty, value: i % numElts + numElts));
530
531 // This should shuffle between cmp (first vector) and null (second vector)
532 mlir::Value nullVec = builder.getNullValue(ty: cmp.getType(), loc);
533 cmp = builder.createVecShuffle(loc, vec1: cmp, vec2: nullVec, maskAttrs: indices);
534 }
535 return builder.createBitcast(src: cmp, newTy: builder.getUIntNTy(n: std::max(a: numElts, b: 8U)));
536}
537
538// TODO: The cgf parameter should be removed when all the NYI cases are
539// implemented.
540static std::optional<mlir::Value>
541emitX86MaskedCompare(CIRGenBuilderTy &builder, unsigned cc, bool isSigned,
542 ArrayRef<mlir::Value> ops, mlir::Location loc) {
543 assert((ops.size() == 2 || ops.size() == 4) &&
544 "Unexpected number of arguments");
545 unsigned numElts = cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize();
546 mlir::Value cmp;
547 if (cc == 3) {
548 cmp = builder.getNullValue(
549 ty: cir::VectorType::getFixed(elementType: builder.getSIntNTy(n: 1), size: numElts), loc);
550 } else if (cc == 7) {
551 cir::VectorType resultTy =
552 cir::VectorType::getFixed(elementType: builder.getSIntNTy(n: 1), size: numElts);
553 llvm::APInt allOnes = llvm::APInt::getAllOnes(numBits: 1);
554 cmp = cir::VecSplatOp::create(
555 builder, location: loc, result: resultTy,
556 value: builder.getConstAPInt(loc, typ: builder.getSIntNTy(n: 1), val: allOnes));
557 } else {
558 cir::CmpOpKind pred;
559 switch (cc) {
560 default:
561 llvm_unreachable("Unknown condition code");
562 case 0:
563 pred = cir::CmpOpKind::eq;
564 break;
565 case 1:
566 pred = cir::CmpOpKind::lt;
567 break;
568 case 2:
569 pred = cir::CmpOpKind::le;
570 break;
571 case 4:
572 pred = cir::CmpOpKind::ne;
573 break;
574 case 5:
575 pred = cir::CmpOpKind::ge;
576 break;
577 case 6:
578 pred = cir::CmpOpKind::gt;
579 break;
580 }
581
582 auto resultTy = cir::VectorType::getFixed(elementType: builder.getSIntNTy(n: 1), size: numElts);
583 cmp = cir::VecCmpOp::create(builder, location: loc, result: resultTy, kind: pred, lhs: ops[0], rhs: ops[1]);
584 }
585
586 mlir::Value maskIn;
587 if (ops.size() == 4)
588 maskIn = ops[3];
589
590 return emitX86MaskedCompareResult(builder, cmp, numElts, maskIn, loc);
591}
592
593// TODO: The cgf parameter should be removed when all the NYI cases are
594// implemented.
595static std::optional<mlir::Value> emitX86ConvertToMask(CIRGenFunction &cgf,
596 CIRGenBuilderTy &builder,
597 mlir::Value in,
598 mlir::Location loc) {
599 cir::ConstantOp zero = builder.getNullValue(ty: in.getType(), loc);
600 return emitX86MaskedCompare(builder, cc: 1, isSigned: true, ops: {in, zero}, loc);
601}
602
603static std::optional<mlir::Value> emitX86SExtMask(CIRGenBuilderTy &builder,
604 mlir::Value op,
605 mlir::Type dstTy,
606 mlir::Location loc) {
607 unsigned numberOfElements = cast<cir::VectorType>(Val&: dstTy).getFixedSize();
608 mlir::Value mask = getMaskVecValue(builder, loc, mask: op, numElems: numberOfElements);
609
610 return builder.createCast(loc, kind: cir::CastKind::integral, src: mask, newTy: dstTy);
611}
612
613static mlir::Value emitVecInsert(CIRGenBuilderTy &builder, mlir::Location loc,
614 mlir::Value vec, mlir::Value value,
615 mlir::Value indexOp) {
616 unsigned numElts = cast<cir::VectorType>(Val: vec.getType()).getFixedSize();
617
618 uint64_t index =
619 indexOp.getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue();
620
621 index &= numElts - 1;
622
623 cir::ConstantOp indexVal = builder.getUInt64(c: index, loc);
624
625 return cir::VecInsertOp::create(builder, location: loc, vec, value, index: indexVal);
626}
627
628static mlir::Value emitX86FunnelShift(CIRGenBuilderTy &builder,
629 mlir::Location location, mlir::Value &op0,
630 mlir::Value &op1, mlir::Value &amt,
631 bool isRight) {
632 mlir::Type op0Ty = op0.getType();
633
634 // Amount may be scalar immediate, in which case create a splat vector.
635 // Funnel shifts amounts are treated as modulo and types are all power-of-2
636 // so we only care about the lowest log2 bits anyway.
637 if (amt.getType() != op0Ty) {
638 auto vecTy = mlir::cast<cir::VectorType>(Val&: op0Ty);
639 uint64_t numElems = vecTy.getFixedSize();
640
641 auto amtTy = mlir::cast<cir::IntType>(Val: amt.getType());
642 auto vecElemTy = mlir::cast<cir::IntType>(Val: vecTy.getElementType());
643
644 // If signed, cast to the same width but unsigned first to
645 // ensure zero-extension when casting to a bigger unsigned `vecElemeTy`.
646 if (amtTy.isSigned()) {
647 cir::IntType unsignedAmtTy = builder.getUIntNTy(n: amtTy.getWidth());
648 amt = builder.createIntCast(src: amt, newTy: unsignedAmtTy);
649 }
650 cir::IntType unsignedVecElemType = builder.getUIntNTy(n: vecElemTy.getWidth());
651 amt = builder.createIntCast(src: amt, newTy: unsignedVecElemType);
652 amt = cir::VecSplatOp::create(
653 builder, location,
654 result: cir::VectorType::getFixed(elementType: unsignedVecElemType, size: numElems), value: amt);
655 }
656
657 const StringRef intrinsicName = isRight ? "fshr" : "fshl";
658 return builder.emitIntrinsicCallOp(loc: location, str: intrinsicName, resTy: op0Ty,
659 op: mlir::ValueRange{op0, op1, amt});
660}
661
662static mlir::Value emitX86Muldq(CIRGenBuilderTy &builder, mlir::Location loc,
663 bool isSigned,
664 SmallVectorImpl<mlir::Value> &ops,
665 unsigned opTypePrimitiveSizeInBits) {
666 mlir::Type ty = cir::VectorType::getFixed(elementType: builder.getSInt64Ty(),
667 size: opTypePrimitiveSizeInBits / 64);
668 mlir::Value lhs = builder.createBitcast(loc, src: ops[0], newTy: ty);
669 mlir::Value rhs = builder.createBitcast(loc, src: ops[1], newTy: ty);
670 if (isSigned) {
671 cir::ConstantOp shiftAmt =
672 builder.getConstant(loc, attr: cir::IntAttr::get(type: builder.getSInt64Ty(), value: 32));
673 cir::VecSplatOp shiftSplatVecOp =
674 cir::VecSplatOp::create(builder, location: loc, result: ty, value: shiftAmt.getResult());
675 mlir::Value shiftSplatValue = shiftSplatVecOp.getResult();
676 // In CIR, right-shift operations are automatically lowered to either an
677 // arithmetic or logical shift depending on the operand type. The purpose
678 // of the shifts here is to propagate the sign bit of the 32-bit input
679 // into the upper bits of each vector lane.
680 lhs = builder.createShift(loc, lhs, rhs: shiftSplatValue, isShiftLeft: true);
681 lhs = builder.createShift(loc, lhs, rhs: shiftSplatValue, isShiftLeft: false);
682 rhs = builder.createShift(loc, lhs: rhs, rhs: shiftSplatValue, isShiftLeft: true);
683 rhs = builder.createShift(loc, lhs: rhs, rhs: shiftSplatValue, isShiftLeft: false);
684 } else {
685 cir::ConstantOp maskScalar = builder.getConstant(
686 loc, attr: cir::IntAttr::get(type: builder.getSInt64Ty(), value: 0xffffffff));
687 cir::VecSplatOp mask =
688 cir::VecSplatOp::create(builder, location: loc, result: ty, value: maskScalar.getResult());
689 // Clear the upper bits
690 lhs = builder.createAnd(loc, lhs, rhs: mask);
691 rhs = builder.createAnd(loc, lhs: rhs, rhs: mask);
692 }
693 return builder.createMul(loc, lhs, rhs);
694}
695
696// Convert f16 half values to floats.
697static mlir::Value emitX86CvtF16ToFloatExpr(CIRGenBuilderTy &builder,
698 mlir::Location loc,
699 llvm::ArrayRef<mlir::Value> ops,
700 mlir::Type dstTy) {
701 assert((ops.size() == 1 || ops.size() == 3 || ops.size() == 4) &&
702 "Unknown cvtph2ps intrinsic");
703
704 // If the SAE intrinsic doesn't use default rounding then we can't upgrade.
705 if (ops.size() == 4) {
706 auto constOp = ops[3].getDefiningOp<cir::ConstantOp>();
707 assert(constOp && "Expected constant operand");
708 if (constOp.getIntValue().getZExtValue() != 4) {
709 return builder.emitIntrinsicCallOp(loc, str: "x86.avx512.mask.vcvtph2ps.512",
710 resTy: dstTy, op&: ops);
711 }
712 }
713
714 unsigned numElts = cast<cir::VectorType>(Val&: dstTy).getFixedSize();
715 mlir::Value src = ops[0];
716
717 // Extract the subvector
718 if (numElts != cast<cir::VectorType>(Val: src.getType()).getFixedSize()) {
719 assert(numElts == 4 && "Unexpected vector size");
720 src = builder.createVecShuffle(loc, vec1: src, mask: {0, 1, 2, 3});
721 }
722
723 // Bitcast from vXi16 to vXf16.
724 cir::VectorType halfTy = cir::VectorType::getFixed(
725 elementType: cir::FP16Type::get(ctx: builder.getContext()), size: numElts);
726
727 src = builder.createCast(kind: cir::CastKind::bitcast, src, newTy: halfTy);
728
729 // Perform the fp-extension
730 mlir::Value res = builder.createCast(kind: cir::CastKind::floating, src, newTy: dstTy);
731
732 if (ops.size() >= 3)
733 res = emitX86Select(builder, loc, mask: ops[2], op0: res, op1: ops[1]);
734 return res;
735}
736
737static mlir::Value emitX86vpcom(CIRGenBuilderTy &builder, mlir::Location loc,
738 llvm::SmallVector<mlir::Value> ops,
739 bool isSigned) {
740 mlir::Value op0 = ops[0];
741 mlir::Value op1 = ops[1];
742
743 cir::VectorType ty = cast<cir::VectorType>(Val: op0.getType());
744 cir::IntType elementTy = cast<cir::IntType>(Val: ty.getElementType());
745
746 uint64_t imm = CIRGenFunction::getZExtIntValueFromConstOp(val: ops[2]) & 0x7;
747
748 cir::CmpOpKind pred;
749 switch (imm) {
750 case 0x0:
751 pred = cir::CmpOpKind::lt;
752 break;
753 case 0x1:
754 pred = cir::CmpOpKind::le;
755 break;
756 case 0x2:
757 pred = cir::CmpOpKind::gt;
758 break;
759 case 0x3:
760 pred = cir::CmpOpKind::ge;
761 break;
762 case 0x4:
763 pred = cir::CmpOpKind::eq;
764 break;
765 case 0x5:
766 pred = cir::CmpOpKind::ne;
767 break;
768 case 0x6:
769 return builder.getNullValue(ty, loc); // FALSE
770 case 0x7: {
771 llvm::APInt allOnes = llvm::APInt::getAllOnes(numBits: elementTy.getWidth());
772 return cir::VecSplatOp::create(
773 builder, location: loc, result: ty,
774 value: builder.getConstAPInt(loc, typ: elementTy, val: allOnes)); // TRUE
775 }
776 default:
777 llvm_unreachable("Unexpected XOP vpcom/vpcomu predicate");
778 }
779
780 if ((!isSigned && elementTy.isSigned()) ||
781 (isSigned && elementTy.isUnsigned())) {
782 elementTy = elementTy.isSigned() ? builder.getUIntNTy(n: elementTy.getWidth())
783 : builder.getSIntNTy(n: elementTy.getWidth());
784 ty = cir::VectorType::get(element_type: elementTy, size: ty.getSize());
785 op0 = builder.createBitcast(src: op0, newTy: ty);
786 op1 = builder.createBitcast(src: op1, newTy: ty);
787 }
788
789 return builder.createVecCompare(loc, kind: pred, lhs: op0, rhs: op1);
790}
791
792static mlir::Value emitX86Fpclass(CIRGenBuilderTy &builder, mlir::Location loc,
793 unsigned builtinID,
794 SmallVectorImpl<mlir::Value> &ops) {
795 unsigned numElts = cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize();
796 mlir::Value maskIn = ops[2];
797 ops.erase(CI: ops.begin() + 2);
798
799 StringRef intrinsicName;
800 switch (builtinID) {
801 default:
802 llvm_unreachable("Unsupported fpclass builtin");
803 case X86::BI__builtin_ia32_vfpclassbf16128_mask:
804 intrinsicName = "x86.avx10.fpclass.bf16.128";
805 break;
806 case X86::BI__builtin_ia32_vfpclassbf16256_mask:
807 intrinsicName = "x86.avx10.fpclass.bf16.256";
808 break;
809 case X86::BI__builtin_ia32_vfpclassbf16512_mask:
810 intrinsicName = "x86.avx10.fpclass.bf16.512";
811 break;
812 case X86::BI__builtin_ia32_fpclassph128_mask:
813 intrinsicName = "x86.avx512fp16.fpclass.ph.128";
814 break;
815 case X86::BI__builtin_ia32_fpclassph256_mask:
816 intrinsicName = "x86.avx512fp16.fpclass.ph.256";
817 break;
818 case X86::BI__builtin_ia32_fpclassph512_mask:
819 intrinsicName = "x86.avx512fp16.fpclass.ph.512";
820 break;
821 case X86::BI__builtin_ia32_fpclassps128_mask:
822 intrinsicName = "x86.avx512.fpclass.ps.128";
823 break;
824 case X86::BI__builtin_ia32_fpclassps256_mask:
825 intrinsicName = "x86.avx512.fpclass.ps.256";
826 break;
827 case X86::BI__builtin_ia32_fpclassps512_mask:
828 intrinsicName = "x86.avx512.fpclass.ps.512";
829 break;
830 case X86::BI__builtin_ia32_fpclasspd128_mask:
831 intrinsicName = "x86.avx512.fpclass.pd.128";
832 break;
833 case X86::BI__builtin_ia32_fpclasspd256_mask:
834 intrinsicName = "x86.avx512.fpclass.pd.256";
835 break;
836 case X86::BI__builtin_ia32_fpclasspd512_mask:
837 intrinsicName = "x86.avx512.fpclass.pd.512";
838 break;
839 }
840
841 auto cmpResultTy = cir::VectorType::getFixed(elementType: builder.getSIntNTy(n: 1), size: numElts);
842 mlir::Value fpclass =
843 builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: cmpResultTy, op&: ops);
844 return emitX86MaskedCompareResult(builder, cmp: fpclass, numElts, maskIn, loc);
845}
846
847static mlir::Value emitX86Aes(CIRGenBuilderTy &builder, mlir::Location loc,
848 llvm::StringRef intrinsicName, mlir::Type retType,
849 llvm::ArrayRef<mlir::Value> ops) {
850 // Create return struct type and call intrinsic function.
851 mlir::Type vecType =
852 mlir::cast<cir::PointerType>(Val: ops[0].getType()).getPointee();
853 mlir::Type rstMembers[] = {retType, vecType};
854 cir::RecordType rstRecTy =
855 builder.getAnonRecordTy(members: rstMembers, /*packed=*/false,
856 memberKinds: cir::RecordType::getAllDataKinds(members: rstMembers));
857 mlir::Value rstValueRec = builder.emitIntrinsicCallOp(
858 loc, str: intrinsicName, resTy: rstRecTy, op: mlir::ValueRange{ops[1], ops[2]});
859
860 // Extract the first return value and truncate it to 1 bit, then cast result
861 // to bool value.
862 mlir::Value flag =
863 cir::ExtractMemberOp::create(builder, location: loc, record: rstValueRec, /*index=*/0);
864 mlir::Value flagBit0 = builder.createCast(loc, kind: cir::CastKind::integral, src: flag,
865 newTy: builder.getUIntNTy(n: 1));
866 mlir::Value succ = builder.createCast(loc, kind: cir::CastKind::int_to_bool,
867 src: flagBit0, newTy: builder.getBoolTy());
868
869 // Extract the second return value, store it to output address if success.
870 mlir::Value out =
871 cir::ExtractMemberOp::create(builder, location: loc, record: rstValueRec, /*index=*/1);
872 Address outAddr(ops[0], /*align=*/CharUnits::fromQuantity(Quantity: 16));
873 cir::IfOp::create(
874 builder, location: loc, cond: succ, /*withElseRegion=*/true,
875 /*thenBuilder=*/
876 [&](mlir::OpBuilder &b, mlir::Location) {
877 builder.createStore(loc, val: out, dst: outAddr);
878 builder.createYield(loc);
879 },
880 /*elseBuilder=*/
881 [&](mlir::OpBuilder &b, mlir::Location) {
882 mlir::Value zero = builder.getNullValue(ty: vecType, loc);
883 builder.createStore(loc, val: zero, dst: outAddr);
884 builder.createYield(loc);
885 });
886
887 return cir::ExtractMemberOp::create(builder, location: loc, record: rstValueRec, /*index=*/0);
888}
889
890static mlir::Value emitX86Aeswide(CIRGenBuilderTy &builder, mlir::Location loc,
891 llvm::StringRef intrinsicName,
892 mlir::Type retType,
893 llvm::ArrayRef<mlir::Value> ops) {
894 mlir::Type vecType =
895 mlir::cast<cir::PointerType>(Val: ops[1].getType()).getPointee();
896
897 // Create struct for return type and load input arguments, then call
898 // intrinsic function.
899 mlir::Type recTypes[9] = {retType, vecType, vecType, vecType, vecType,
900 vecType, vecType, vecType, vecType};
901 mlir::Value arguments[9];
902 arguments[0] = ops[2];
903 for (int i = 0; i < 8; i++) {
904 // Loading each vector argument from input address.
905 cir::ConstantOp idx = builder.getUInt32(c: i, loc);
906 mlir::Value nextInElePtr =
907 builder.getArrayElement(arrayLocBegin: loc, arrayLocEnd: loc, arrayPtr: ops[1], eltTy: vecType, idx,
908 /*shouldDecay=*/false);
909 arguments[i + 1] =
910 builder.createAlignedLoad(loc, ty: vecType, ptr: nextInElePtr,
911 /*align=*/CharUnits::fromQuantity(Quantity: 16));
912 }
913 cir::RecordType rstRecTy = builder.getAnonRecordTy(
914 members: recTypes, /*packed=*/false, memberKinds: cir::RecordType::getAllDataKinds(members: recTypes));
915 mlir::Value rstValueRec =
916 builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: rstRecTy, op&: arguments);
917
918 // Extract the first return value and truncate it to 1 bit, then cast result
919 // to bool value.
920 mlir::Value flag =
921 cir::ExtractMemberOp::create(builder, location: loc, record: rstValueRec, /*index=*/0);
922 mlir::Value flagBit0 = builder.createCast(loc, kind: cir::CastKind::integral, src: flag,
923 newTy: builder.getUIntNTy(n: 1));
924 mlir::Value succ = builder.createCast(loc, kind: cir::CastKind::int_to_bool,
925 src: flagBit0, newTy: builder.getBoolTy());
926
927 // Extract other return values, store those to output address if success.
928 cir::IfOp::create(
929 builder, location: loc, cond: succ, /*withElseRegion=*/true,
930 /*thenBuilder=*/
931 [&](mlir::OpBuilder &b, mlir::Location) {
932 for (int i = 0; i < 8; i++) {
933 mlir::Value out =
934 cir::ExtractMemberOp::create(builder, location: loc, record: rstValueRec,
935 /*index=*/i + 1);
936 cir::ConstantOp idx = builder.getUInt32(c: i, loc);
937 mlir::Value nextOutEleAddr =
938 builder.getArrayElement(arrayLocBegin: loc, arrayLocEnd: loc, arrayPtr: ops[0], eltTy: vecType, idx,
939 /*shouldDecay=*/false);
940 Address outAddr(nextOutEleAddr,
941 /*align=*/CharUnits::fromQuantity(Quantity: 16));
942 builder.createStore(loc, val: out, dst: outAddr);
943 }
944 builder.createYield(loc);
945 },
946 /*elseBuilder=*/
947 [&](mlir::OpBuilder &b, mlir::Location) {
948 mlir::Value zero = builder.getNullValue(ty: vecType, loc);
949 for (int i = 0; i < 8; i++) {
950 cir::ConstantOp idx = builder.getUInt32(c: i, loc);
951 mlir::Value nextOutEleAddr =
952 builder.getArrayElement(arrayLocBegin: loc, arrayLocEnd: loc, arrayPtr: ops[0], eltTy: vecType, idx,
953 /*shouldDecay=*/false);
954 Address outAddr(nextOutEleAddr,
955 /*align=*/CharUnits::fromQuantity(Quantity: 16));
956 builder.createStore(loc, val: zero, dst: outAddr);
957 }
958 builder.createYield(loc);
959 });
960
961 return cir::ExtractMemberOp::create(builder, location: loc, record: rstValueRec, /*index=*/0);
962}
963
964static mlir::Value emitX86MaskedLoad(CIRGenBuilderTy &builder,
965 ArrayRef<mlir::Value> ops,
966 llvm::Align alignment,
967 mlir::Location loc) {
968 mlir::Type ty = ops[1].getType();
969 mlir::Value ptr = ops[0];
970 mlir::Value maskVec = getMaskVecValue(
971 builder, loc, mask: ops[2], numElems: cast<cir::VectorType>(Val&: ty).getFixedSize());
972
973 return builder.createMaskedLoad(loc, ty, ptr, alignment, mask: maskVec, passThru: ops[1]);
974}
975
976static mlir::Value emitX86VPerm2f128(CIRGenBuilderTy &builder,
977 mlir::Location loc,
978 llvm::SmallVector<mlir::Value> ops) {
979 auto inputType = cast<cir::VectorType>(Val: ops[0].getType());
980 assert(!inputType.isScalable() &&
981 "This is only intended for fixed-width vectors");
982
983 const uint8_t imm = CIRGenFunction::getZExtIntValueFromConstOp(val: ops[2]);
984 mlir::Value zeroVec = builder.getZero(loc, ty: inputType);
985
986 // If both lanes are zero, return a zero result.
987 if ((imm & 0x80) && (imm & 0x08))
988 return zeroVec;
989
990 mlir::Value lanes[2];
991 llvm::SmallVector<mlir::Attribute, 64> mask;
992
993 cir::IntType i32Ty = builder.getSInt32Ty();
994 const unsigned numElts = inputType.getFixedSize();
995 // We must evaluated each lane(128 bits) separetely
996 for (auto lane : llvm::seq(Begin: 0, End: 2)) {
997 bool isZeroBit = imm & (1 << ((lane * 4) + 3)),
998 isSourceB = imm & (1 << ((lane * 4) + 1)),
999 isUpperHalf = imm & (1 << (lane * 4));
1000
1001 // Determine the source for this lane
1002 if (isZeroBit)
1003 lanes[lane] = zeroVec;
1004 else
1005 lanes[lane] = isSourceB ? ops[1] : ops[0];
1006
1007 // We need to built the shuffle mask selecting the right half
1008 for (auto elt : llvm::seq(Begin: 0u, End: numElts / 2u)) {
1009 unsigned idx = (lane * numElts) + elt;
1010 if (isUpperHalf)
1011 idx += numElts / 2;
1012 mask.push_back(Elt: cir::IntAttr::get(type: i32Ty, value: idx));
1013 }
1014 }
1015
1016 return builder.createVecShuffle(loc, vec1: lanes[0], vec2: lanes[1], maskAttrs: mask);
1017}
1018
1019static mlir::Value emitX86PackedByteShift(CIRGenBuilderTy &builder,
1020 unsigned builtinID,
1021 mlir::Location loc,
1022 llvm::ArrayRef<mlir::Value> ops,
1023 llvm::Boolean isLeftShift) {
1024 auto byteVecType = cast<cir::VectorType>(Val: ops[0].getType());
1025 assert(!byteVecType.isScalable() &&
1026 "This is only intended for fixed-width vectors");
1027
1028 unsigned shiftVal = CIRGenFunction::getZExtIntValueFromConstOp(val: ops[1]) & 0xFF;
1029 mlir::Value zeroVector = builder.getZero(loc, ty: byteVecType);
1030
1031 // If pslldq is shifting the vector more than 15 bytes, emit zero.
1032 // This matches the hardware behavior where shifting by 16+ bytes
1033 // clears the entire 128-bit lane.
1034 if (shiftVal >= 16)
1035 return zeroVector;
1036
1037 uint64_t numElts = byteVecType.getFixedSize();
1038 assert(numElts % 16 == 0 && "Expected a multiple of 16");
1039
1040 llvm::SmallVector<int64_t, 64> shuffleMask;
1041
1042 constexpr unsigned laneSize = 16;
1043 const int switchOperand = numElts - laneSize;
1044
1045 // 256/512-bit pslldq/psrldq operates on 128-bit lanes so we need to
1046 // handle that
1047 for (auto laneOffset = 0ull; laneOffset < numElts; laneOffset += laneSize) {
1048 for (auto elt : llvm::seq<unsigned>(Begin: 0, End: laneSize)) {
1049 unsigned idx =
1050 isLeftShift ? (numElts + elt - shiftVal) : (elt + shiftVal);
1051
1052 bool isZeroPadding = isLeftShift ? (idx < numElts) : (idx >= laneSize);
1053 if (isZeroPadding)
1054 idx += isLeftShift ? (-switchOperand) : switchOperand;
1055
1056 shuffleMask.push_back(Elt: idx + laneOffset);
1057 }
1058 }
1059
1060 // Perform the shuffle
1061 // (left concatenating zeros on left, right concatenating zeros on right)
1062 auto [firstOperand, secondOperand] = isLeftShift
1063 ? std::make_pair(x&: zeroVector, y: ops[0])
1064 : std::make_pair(x: ops[0], y&: zeroVector);
1065
1066 // Mask the result using circular arithmetic on concatenated buffer
1067 mlir::Value shuffleResult =
1068 builder.createVecShuffle(loc, vec1: firstOperand, vec2: secondOperand, mask: shuffleMask);
1069
1070 return shuffleResult;
1071}
1072
1073mlir::Value CIRGenFunction::emitX86CpuIs(const CallExpr *expr) {
1074 const Expr *cpuExpr = expr->getArg(Arg: 0)->IgnoreParenCasts();
1075 StringRef cpuStr = cast<clang::StringLiteral>(Val: cpuExpr)->getString();
1076 return emitX86CpuIs(loc: getLoc(srcLoc: expr->getExprLoc()), cpuStr);
1077}
1078
1079cir::GetGlobalOp CIRGenFunction::createGetCpuModel(mlir::Location loc) {
1080 mlir::Type u32 = builder.getUInt32Ty();
1081 auto cpuModel =
1082 mlir::dyn_cast_or_null<cir::GlobalOp>(Val: cgm.getGlobalValue(ref: "__cpu_model"));
1083
1084 if (!cpuModel) {
1085 // Matching the struct layout from the compiler-rt/libgcc structure that is
1086 // filled in:
1087 // unsigned int __cpu_vendor;
1088 // unsigned int __cpu_type;
1089 // unsigned int __cpu_subtype;
1090 // unsigned int __cpu_features[1];
1091 mlir::Type tys[] = {u32, u32, u32, cir::ArrayType::get(elementType: u32, size: 1)};
1092 mlir::Type modelTy = builder.getAnonRecordTy(
1093 members: tys, /*packed=*/false, memberKinds: cir::RecordType::getAllDataKinds(members: tys));
1094 cpuModel =
1095 cgm.createGlobalOp(loc, name: "__cpu_model", t: modelTy, /*isConstant=*/false);
1096 cpuModel.setDsoLocal(true);
1097 }
1098
1099 return cir::GetGlobalOp::create(builder, location: loc,
1100 addr: builder.getPointerTo(ty: cpuModel.getSymType()),
1101 name: cpuModel.getSymName());
1102}
1103
1104cir::GetGlobalOp CIRGenFunction::createGetCpuFeatures2(mlir::Location loc) {
1105 mlir::Type u32 = builder.getUInt32Ty();
1106 auto cpuFeatures2 = mlir::dyn_cast_or_null<cir::GlobalOp>(
1107 Val: cgm.getGlobalValue(ref: "__cpu_features2"));
1108
1109 if (!cpuFeatures2) {
1110 // This is just an array of 3 uint32s.
1111 mlir::Type arrTy = cir::ArrayType::get(elementType: u32, size: 3);
1112 cpuFeatures2 =
1113 cgm.createGlobalOp(loc, name: "__cpu_features2", t: arrTy, /*isConstant=*/false);
1114 cpuFeatures2.setDsoLocal(true);
1115 }
1116
1117 return cir::GetGlobalOp::create(
1118 builder, location: loc, addr: builder.getPointerTo(ty: cpuFeatures2.getSymType()),
1119 name: cpuFeatures2.getSymName());
1120}
1121
1122mlir::Value CIRGenFunction::emitX86CpuIs(mlir::Location loc, StringRef cpuStr) {
1123 mlir::Type u32 = builder.getUInt32Ty();
1124 // Calculate the index needed to access the correct field based on the
1125 // range. ABI_VALUE matches with compiler-rt/libgcc values.
1126 auto [fieldName, index, value] =
1127 llvm::StringSwitch<std::tuple<llvm::StringLiteral, unsigned, unsigned>>(
1128 cpuStr)
1129#define X86_VENDOR(ENUM, STRING, ABI_VALUE) \
1130 .Case(STRING, {"__cpu_vendor", 0u, ABI_VALUE})
1131#define X86_CPU_TYPE(ENUM, STR, ABI_VALUE) \
1132 .Case(STR, {"__cpu_type", 1u, ABI_VALUE})
1133#define X86_CPU_SUBTYPE(ENUM, STR, ABI_VALUE) \
1134 .Case(STR, {"__cpu_subtype", 2u, ABI_VALUE})
1135#include "llvm/TargetParser/X86TargetParser.def"
1136 .Default(Value: {"", 0, 0});
1137 assert(value != 0 && "Invalid CPUStr passed to CpuIs");
1138
1139 cir::GetGlobalOp getCpuModel = createGetCpuModel(loc);
1140
1141 // Note: the StringSwitch above ONLY has the ability to get the first 3
1142 // fields, so we don't have to worry about it being the array field. So we
1143 // can continue assuming everything is an int.
1144 cir::GetMemberOp cpuValuePtr = builder.createGetMember(
1145 loc, resultTy: builder.getPointerTo(ty: u32), base: getCpuModel, name: fieldName, index);
1146 cir::LoadOp getVal = builder.createAlignedLoad(loc, ty: u32, ptr: cpuValuePtr,
1147 align: CharUnits::fromQuantity(Quantity: 4));
1148
1149 return cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::eq, lhs: getVal,
1150 rhs: builder.getUInt32(c: value, loc));
1151}
1152
1153mlir::Value CIRGenFunction::emitX86CpuSupports(const CallExpr *expr) {
1154 const Expr *featureExpr = expr->getArg(Arg: 0)->IgnoreParenCasts();
1155 StringRef featureStr = cast<StringLiteral>(Val: featureExpr)->getString();
1156 if (!getContext().getTargetInfo().validateCpuSupports(Name: featureStr))
1157 return builder.getFalse(loc: getLoc(srcLoc: expr->getExprLoc()));
1158 return emitX86CpuSupports(loc: getLoc(srcLoc: expr->getExprLoc()), FeatureStrs: featureStr);
1159}
1160
1161mlir::Value
1162CIRGenFunction::emitX86CpuSupports(mlir::Location loc,
1163 ArrayRef<StringRef> featureStrs) {
1164 return emitX86CpuSupports(loc, FeatureMask: llvm::X86::getCpuSupportsMask(FeatureStrs: featureStrs));
1165}
1166
1167mlir::Value
1168CIRGenFunction::emitX86CpuSupports(mlir::Location loc,
1169 std::array<uint32_t, 4> featureMask) {
1170 mlir::Type u32 = builder.getUInt32Ty();
1171 mlir::Value result;
1172
1173 auto addCondition = [&](unsigned maskVal, mlir::Value features) {
1174 cir::ConstantOp mask = builder.getUInt32(c: maskVal, loc);
1175 mlir::Value bitset = builder.createAnd(loc, lhs: features, rhs: mask);
1176 mlir::Value cmp =
1177 cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::eq, lhs: bitset, rhs: mask);
1178
1179 if (result)
1180 result = builder.createAnd(loc, lhs: result, rhs: cmp);
1181 else
1182 result = cmp;
1183 };
1184
1185 // only check the '__cpu_features[0]' if we have a non-zero value. A zero
1186 // here means to JUST check __cpu_features2.
1187 if (featureMask[0] != 0) {
1188 cir::GetGlobalOp getCpuModel = createGetCpuModel(loc);
1189 mlir::Type u32 = builder.getUInt32Ty();
1190 auto arrTy = cir::ArrayType::get(elementType: u32, size: 1);
1191
1192 // Pick out the __cpu_features field, the 4th field in the struct.
1193 cir::GetMemberOp cpuFeatArrPtr = builder.createGetMember(
1194 loc, resultTy: builder.getPointerTo(ty: arrTy), base: getCpuModel, name: "__cpu_features", index: 3);
1195
1196 mlir::Value cpuFeatVal = builder.getArrayElement(
1197 arrayLocBegin: loc, arrayLocEnd: loc, arrayPtr: cpuFeatArrPtr, eltTy: arrTy,
1198 /*idx=*/builder.getUInt32(c: 0, loc), /*shouldDecay=*/true,
1199 flags: cir::PtrOffsetFlags::InBounds);
1200 cir::LoadOp features = builder.createAlignedLoad(
1201 loc, ty: u32, ptr: cpuFeatVal, align: CharUnits::fromQuantity(Quantity: 4));
1202
1203 addCondition(featureMask[0], features);
1204 }
1205
1206 // the 0th index is looked up in the __cpu_model field, the rest come from an
1207 // array.
1208 cir::GetGlobalOp getCpuFeatures2; // = createGetCpuFeatures2(loc);
1209 for (int i = 1; i != 4; ++i) {
1210 const uint32_t val = featureMask[i];
1211 if (!val)
1212 continue;
1213 if (!getCpuFeatures2)
1214 getCpuFeatures2 = createGetCpuFeatures2(loc);
1215 mlir::Value getArrayElt = builder.getArrayElement(
1216 arrayLocBegin: loc, arrayLocEnd: loc, arrayPtr: getCpuFeatures2,
1217 eltTy: cast<cir::PointerType>(Val: getCpuFeatures2.getType()).getPointee(),
1218 idx: builder.getUInt32(/*index=*/c: i - 1, loc),
1219 /*shouldDecay=*/true, flags: cir::PtrOffsetFlags::InBounds);
1220 cir::LoadOp features = builder.createAlignedLoad(
1221 loc, ty: u32, ptr: getArrayElt, align: CharUnits::fromQuantity(Quantity: 4));
1222
1223 addCondition(val, features);
1224 }
1225
1226 return result;
1227}
1228
1229mlir::Value CIRGenFunction::emitX86CpuInit(mlir::Location loc) {
1230 cir::FuncOp initFunc =
1231 cgm.createRuntimeFunction(ty: builder.getVoidFnTy(), name: "__cpu_indicator_init");
1232 initFunc.setDsoLocal(true);
1233 assert(!cir::MissingFeatures::setDLLStorageClass());
1234
1235 return builder.createCallOp(loc, callee: initFunc, operands: {}).getResult();
1236}
1237
1238mlir::Value CIRGenFunction::emitX86MSVCIntrin(MSVCIntrin builtinID,
1239 const CallExpr *E) {
1240 switch (builtinID) {
1241 case MSVCIntrin::_BitScanForward:
1242 case MSVCIntrin::_BitScanReverse: {
1243 Address indexAddress(emitPointerWithAlignment(expr: E->getArg(Arg: 0)));
1244 mlir::Value argValue = emitScalarExpr(e: E->getArg(Arg: 1));
1245
1246 mlir::Location loc = getLoc(srcLoc: E->getExprLoc());
1247 mlir::Type argType = argValue.getType();
1248 mlir::Type indexType = indexAddress.getElementType();
1249 mlir::Type resultType = convertType(t: E->getType());
1250
1251 mlir::Value argZero = builder.getNullValue(ty: argType, loc);
1252 mlir::Value isZero =
1253 cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::eq, lhs: argValue, rhs: argZero);
1254 cir::IfOp::create(
1255 builder, location: loc, cond: isZero, /*withElseRegion=*/true,
1256 thenBuilder: [&](mlir::OpBuilder &b, mlir::Location) { builder.createYield(loc); },
1257 elseBuilder: [&](mlir::OpBuilder &b, mlir::Location) {
1258 if (builtinID == MSVCIntrin::_BitScanForward) {
1259 mlir::Value zeroCount =
1260 cir::BitCtzOp::create(builder, location: loc, input: argValue, poison_zero: true);
1261 mlir::Value castCount = builder.createCast(
1262 loc, kind: cir::CastKind::integral, src: zeroCount, newTy: indexType);
1263 builder.createStore(loc, val: castCount, dst: indexAddress, isVolatile: false);
1264 } else {
1265 unsigned argWidth = mlir::cast<cir::IntType>(Val&: argType).getWidth();
1266 mlir::Value argTypeLastIndex =
1267 builder.getConstantInt(loc, ty: indexType, value: argWidth - 1);
1268
1269 mlir::Value zeroCount =
1270 cir::BitClzOp::create(builder, location: loc, input: argValue, poison_zero: true);
1271 mlir::Value castCount = builder.createCast(
1272 loc, kind: cir::CastKind::integral, src: zeroCount, newTy: indexType);
1273 mlir::Value index =
1274 builder.createNSWSub(loc, lhs: argTypeLastIndex, rhs: castCount);
1275 builder.createStore(loc, val: index, dst: indexAddress, isVolatile: false);
1276 }
1277 builder.createYield(loc);
1278 });
1279 mlir::Value notZero = builder.createNot(loc, value: isZero);
1280 return builder.createCast(loc, kind: cir::CastKind::bool_to_int, src: notZero,
1281 newTy: resultType);
1282 }
1283 case MSVCIntrin::_InterlockedAnd:
1284 case MSVCIntrin::_InterlockedExchange:
1285 case MSVCIntrin::_InterlockedExchangeAdd:
1286 case MSVCIntrin::_InterlockedExchangeSub:
1287 case MSVCIntrin::_InterlockedOr:
1288 case MSVCIntrin::_InterlockedXor:
1289 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1290 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1291 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1292 case MSVCIntrin::_InterlockedExchange_acq:
1293 case MSVCIntrin::_InterlockedExchange_rel:
1294 case MSVCIntrin::_InterlockedExchange_nf:
1295 case MSVCIntrin::_InterlockedCompareExchange:
1296 case MSVCIntrin::_InterlockedCompareExchange_acq:
1297 case MSVCIntrin::_InterlockedCompareExchange_rel:
1298 case MSVCIntrin::_InterlockedCompareExchange_nf:
1299 case MSVCIntrin::_InterlockedCompareExchange128:
1300 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1301 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1302 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1303 case MSVCIntrin::_InterlockedOr_acq:
1304 case MSVCIntrin::_InterlockedOr_rel:
1305 case MSVCIntrin::_InterlockedOr_nf:
1306 case MSVCIntrin::_InterlockedXor_acq:
1307 case MSVCIntrin::_InterlockedXor_rel:
1308 case MSVCIntrin::_InterlockedXor_nf:
1309 case MSVCIntrin::_InterlockedAnd_acq:
1310 case MSVCIntrin::_InterlockedAnd_rel:
1311 case MSVCIntrin::_InterlockedAnd_nf:
1312 case MSVCIntrin::_InterlockedIncrement_acq:
1313 case MSVCIntrin::_InterlockedIncrement_rel:
1314 case MSVCIntrin::_InterlockedIncrement_nf:
1315 case MSVCIntrin::_InterlockedDecrement_acq:
1316 case MSVCIntrin::_InterlockedDecrement_rel:
1317 case MSVCIntrin::_InterlockedDecrement_nf:
1318 case MSVCIntrin::_InterlockedDecrement:
1319 case MSVCIntrin::_InterlockedIncrement:
1320 case MSVCIntrin::__fastfail:
1321 cgm.errorNYI(E->getExprLoc(), "MSVC intrinsic");
1322 return {};
1323 case MSVCIntrin::_ReadWriteBarrier:
1324 cir::AtomicFenceOp::create(
1325 builder, location: getLoc(srcLoc: E->getExprLoc()), ordering: cir::MemOrder::SequentiallyConsistent,
1326 syncscope: cir::SyncScopeKindAttr::get(context: &getMLIRContext(),
1327 value: cir::SyncScopeKind::SingleThread));
1328 return mlir::Value{};
1329 }
1330 llvm_unreachable("Incorrect MSVC intrinsic!");
1331}
1332
1333std::optional<mlir::Value>
1334CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID, const CallExpr *expr) {
1335 if (builtinID == Builtin::BI__builtin_cpu_is)
1336 return emitX86CpuIs(expr);
1337 if (builtinID == Builtin::BI__builtin_cpu_supports)
1338 return emitX86CpuSupports(expr);
1339 if (builtinID == Builtin::BI__builtin_cpu_init)
1340 return emitX86CpuInit(loc: getLoc(srcLoc: expr->getExprLoc()));
1341
1342 // Handle MSVC intrinsics before argument evaluation to prevent double
1343 // evaluation.
1344
1345 if (std::optional<MSVCIntrin> MsvcIntId =
1346 CodeGenUtils::translateX86ToMsvcIntrin(BuiltinID: builtinID))
1347 return emitX86MSVCIntrin(builtinID: *MsvcIntId, E: expr);
1348
1349 // Find out if any arguments are required to be integer constant expressions.
1350 assert(!cir::MissingFeatures::handleBuiltinICEArguments());
1351
1352 // The operands of the builtin call
1353 llvm::SmallVector<mlir::Value> ops;
1354 bool isMaskFCmp = false;
1355
1356 // `ICEArguments` is a bitmap indicating whether the argument at the i-th bit
1357 // is required to be a constant integer expression.
1358 unsigned iceArguments = 0;
1359 ASTContext::GetBuiltinTypeError error;
1360 getContext().GetBuiltinType(ID: builtinID, Error&: error, IntegerConstantArgs: &iceArguments);
1361 assert(error == ASTContext::GE_None && "Error while getting builtin type.");
1362
1363 for (auto [idx, arg] : llvm::enumerate(First: expr->arguments()))
1364 ops.push_back(Elt: emitScalarOrConstFoldImmArg(iceArguments, idx, argExpr: arg));
1365
1366 CIRGenBuilderTy &builder = getBuilder();
1367 mlir::Type voidTy = builder.getVoidTy();
1368
1369 switch (builtinID) {
1370 default:
1371 return std::nullopt;
1372 case X86::BI_mm_clflush:
1373 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()),
1374 str: "x86.sse2.clflush", resTy: voidTy, op&: ops[0]);
1375 case X86::BI_mm_lfence:
1376 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()),
1377 str: "x86.sse2.lfence", resTy: voidTy);
1378 case X86::BI_mm_pause:
1379 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()),
1380 str: "x86.sse2.pause", resTy: voidTy);
1381 case X86::BI_mm_mfence:
1382 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()),
1383 str: "x86.sse2.mfence", resTy: voidTy);
1384 case X86::BI_mm_sfence:
1385 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()),
1386 str: "x86.sse.sfence", resTy: voidTy);
1387 case X86::BI_mm_prefetch:
1388 case X86::BI_m_prefetch:
1389 case X86::BI_m_prefetchw:
1390 return emitPrefetch(cgf&: *this, builtinID, e: expr, ops);
1391 case X86::BI__rdtsc:
1392 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()), str: "x86.rdtsc",
1393 resTy: builder.getUInt64Ty());
1394 case X86::BI__builtin_ia32_rdtscp: {
1395 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
1396 mlir::Type i64Ty = builder.getUInt64Ty();
1397 mlir::Type i32Ty = builder.getUInt32Ty();
1398 mlir::Type members[] = {i64Ty, i32Ty};
1399 mlir::Type structTy = builder.getAnonRecordTy(
1400 members, /*packed=*/false, memberKinds: cir::RecordType::getAllDataKinds(members));
1401 mlir::Value result =
1402 builder.emitIntrinsicCallOp(loc, str: "x86.rdtscp", resTy: structTy);
1403
1404 // Extract and store processor_id (element 1 of the returned struct)
1405 mlir::Value processorId =
1406 cir::ExtractMemberOp::create(builder, location: loc, result: i32Ty, record: result, index: 1);
1407 // ops[0] is the address to store the processor ID
1408 builder.createStore(loc, val: processorId, dst: Address{ops[0], CharUnits::One()});
1409
1410 // Return timestamp (element 0 of the returned struct)
1411 return cir::ExtractMemberOp::create(builder, location: loc, result: i64Ty, record: result, index: 0);
1412 }
1413 case X86::BI__builtin_ia32_roundps:
1414 case X86::BI__builtin_ia32_roundpd:
1415 case X86::BI__builtin_ia32_roundps256:
1416 case X86::BI__builtin_ia32_roundpd256:
1417 case X86::BI__builtin_ia32_roundss:
1418 case X86::BI__builtin_ia32_roundsd:
1419 return emitX86Round(cgf&: *this, builtinID, expr, ops);
1420 case X86::BI__builtin_ia32_lzcnt_u16:
1421 case X86::BI__builtin_ia32_lzcnt_u32:
1422 case X86::BI__builtin_ia32_lzcnt_u64:
1423 return cir::BitClzOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()), input: ops[0],
1424 /*poisonZero=*/poison_zero: false)
1425 .getResult();
1426 case X86::BI__builtin_ia32_tzcnt_u16:
1427 case X86::BI__builtin_ia32_tzcnt_u32:
1428 case X86::BI__builtin_ia32_tzcnt_u64:
1429 return cir::BitCtzOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()), input: ops[0],
1430 /*poisonZero=*/poison_zero: false)
1431 .getResult();
1432 case X86::BI__builtin_ia32_undef128:
1433 case X86::BI__builtin_ia32_undef256:
1434 case X86::BI__builtin_ia32_undef512:
1435 // The x86 definition of "undef" is not the same as the LLVM definition
1436 // (PR32176). We leave optimizing away an unnecessary zero constant to the
1437 // IR optimizer and backend.
1438 // TODO: If we had a "freeze" IR instruction to generate a fixed undef
1439 // value, we should use that here instead of a zero.
1440 return builder.getNullValue(ty: convertType(t: expr->getType()),
1441 loc: getLoc(srcLoc: expr->getExprLoc()));
1442 case X86::BI__builtin_ia32_vec_ext_v4hi:
1443 case X86::BI__builtin_ia32_vec_ext_v16qi:
1444 case X86::BI__builtin_ia32_vec_ext_v8hi:
1445 case X86::BI__builtin_ia32_vec_ext_v4si:
1446 case X86::BI__builtin_ia32_vec_ext_v4sf:
1447 case X86::BI__builtin_ia32_vec_ext_v2di:
1448 case X86::BI__builtin_ia32_vec_ext_v32qi:
1449 case X86::BI__builtin_ia32_vec_ext_v16hi:
1450 case X86::BI__builtin_ia32_vec_ext_v8si:
1451 case X86::BI__builtin_ia32_vec_ext_v4di: {
1452 unsigned numElts = cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize();
1453
1454 uint64_t index = getZExtIntValueFromConstOp(val: ops[1]);
1455 index &= numElts - 1;
1456
1457 cir::ConstantOp indexVal =
1458 builder.getUInt64(c: index, loc: getLoc(srcLoc: expr->getExprLoc()));
1459
1460 // These builtins exist so we can ensure the index is an ICE and in range.
1461 // Otherwise we could just do this in the header file.
1462 return cir::VecExtractOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()),
1463 vec: ops[0], index: indexVal);
1464 }
1465 case X86::BI__builtin_ia32_vec_set_v4hi:
1466 case X86::BI__builtin_ia32_vec_set_v16qi:
1467 case X86::BI__builtin_ia32_vec_set_v8hi:
1468 case X86::BI__builtin_ia32_vec_set_v4si:
1469 case X86::BI__builtin_ia32_vec_set_v2di:
1470 case X86::BI__builtin_ia32_vec_set_v32qi:
1471 case X86::BI__builtin_ia32_vec_set_v16hi:
1472 case X86::BI__builtin_ia32_vec_set_v8si:
1473 case X86::BI__builtin_ia32_vec_set_v4di: {
1474 return emitVecInsert(builder, loc: getLoc(srcLoc: expr->getExprLoc()), vec: ops[0], value: ops[1],
1475 indexOp: ops[2]);
1476 }
1477 case X86::BI__builtin_ia32_kunpckhi:
1478 return emitX86MaskUnpack(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
1479 intrinsicName: "x86.avx512.kunpackb", ops);
1480 case X86::BI__builtin_ia32_kunpcksi:
1481 return emitX86MaskUnpack(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
1482 intrinsicName: "x86.avx512.kunpackw", ops);
1483 case X86::BI__builtin_ia32_kunpckdi:
1484 return emitX86MaskUnpack(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
1485 intrinsicName: "x86.avx512.kunpackd", ops);
1486 case X86::BI_mm_setcsr:
1487 case X86::BI__builtin_ia32_ldmxcsr: {
1488 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
1489 Address tmp = createMemTemp(t: expr->getArg(Arg: 0)->getType(), loc);
1490 builder.createStore(loc, val: ops[0], dst: tmp);
1491 return builder.emitIntrinsicCallOp(loc, str: "x86.sse.ldmxcsr",
1492 resTy: builder.getVoidTy(), op: tmp.getPointer());
1493 }
1494 case X86::BI_mm_getcsr:
1495 case X86::BI__builtin_ia32_stmxcsr: {
1496 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
1497 Address tmp = createMemTemp(t: expr->getType(), loc);
1498 builder.emitIntrinsicCallOp(loc, str: "x86.sse.stmxcsr", resTy: builder.getVoidTy(),
1499 op: tmp.getPointer());
1500 return builder.createLoad(loc, addr: tmp);
1501 }
1502 case X86::BI__builtin_ia32_xsave:
1503 case X86::BI__builtin_ia32_xsave64:
1504 case X86::BI__builtin_ia32_xrstor:
1505 case X86::BI__builtin_ia32_xrstor64:
1506 case X86::BI__builtin_ia32_xsaveopt:
1507 case X86::BI__builtin_ia32_xsaveopt64:
1508 case X86::BI__builtin_ia32_xrstors:
1509 case X86::BI__builtin_ia32_xrstors64:
1510 case X86::BI__builtin_ia32_xsavec:
1511 case X86::BI__builtin_ia32_xsavec64:
1512 case X86::BI__builtin_ia32_xsaves:
1513 case X86::BI__builtin_ia32_xsaves64:
1514 case X86::BI__builtin_ia32_xsetbv:
1515 case X86::BI_xsetbv: {
1516 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
1517 StringRef intrinsicName;
1518 switch (builtinID) {
1519 default:
1520 llvm_unreachable("Unexpected builtin");
1521 case X86::BI__builtin_ia32_xsave:
1522 intrinsicName = "x86.xsave";
1523 break;
1524 case X86::BI__builtin_ia32_xsave64:
1525 intrinsicName = "x86.xsave64";
1526 break;
1527 case X86::BI__builtin_ia32_xrstor:
1528 intrinsicName = "x86.xrstor";
1529 break;
1530 case X86::BI__builtin_ia32_xrstor64:
1531 intrinsicName = "x86.xrstor64";
1532 break;
1533 case X86::BI__builtin_ia32_xsaveopt:
1534 intrinsicName = "x86.xsaveopt";
1535 break;
1536 case X86::BI__builtin_ia32_xsaveopt64:
1537 intrinsicName = "x86.xsaveopt64";
1538 break;
1539 case X86::BI__builtin_ia32_xrstors:
1540 intrinsicName = "x86.xrstors";
1541 break;
1542 case X86::BI__builtin_ia32_xrstors64:
1543 intrinsicName = "x86.xrstors64";
1544 break;
1545 case X86::BI__builtin_ia32_xsavec:
1546 intrinsicName = "x86.xsavec";
1547 break;
1548 case X86::BI__builtin_ia32_xsavec64:
1549 intrinsicName = "x86.xsavec64";
1550 break;
1551 case X86::BI__builtin_ia32_xsaves:
1552 intrinsicName = "x86.xsaves";
1553 break;
1554 case X86::BI__builtin_ia32_xsaves64:
1555 intrinsicName = "x86.xsaves64";
1556 break;
1557 case X86::BI__builtin_ia32_xsetbv:
1558 case X86::BI_xsetbv:
1559 intrinsicName = "x86.xsetbv";
1560 break;
1561 }
1562
1563 // The xsave family of instructions take a 64-bit mask that specifies
1564 // which processor state components to save/restore. The hardware expects
1565 // this mask split into two 32-bit registers: EDX (high 32 bits) and
1566 // EAX (low 32 bits).
1567 mlir::Type i32Ty = builder.getSInt32Ty();
1568
1569 // Mhi = (uint32_t)(ops[1] >> 32) - extract high 32 bits via right shift
1570 cir::ConstantOp shift32 = builder.getSInt64(c: 32, loc);
1571 mlir::Value mhi = builder.createShift(loc, lhs: ops[1], rhs: shift32.getResult(),
1572 /*isShiftLeft=*/false);
1573 mhi = builder.createIntCast(src: mhi, newTy: i32Ty);
1574
1575 // Mlo = (uint32_t)ops[1] - extract low 32 bits by truncation
1576 mlir::Value mlo = builder.createIntCast(src: ops[1], newTy: i32Ty);
1577
1578 return builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: voidTy,
1579 op: mlir::ValueRange{ops[0], mhi, mlo});
1580 }
1581 case X86::BI__builtin_ia32_xgetbv:
1582 case X86::BI_xgetbv:
1583 // xgetbv reads the extended control register specified by ops[0] (ECX)
1584 // and returns the 64-bit value
1585 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()), str: "x86.xgetbv",
1586 resTy: builder.getUInt64Ty(), op&: ops[0]);
1587 case X86::BI__builtin_ia32_storedqudi128_mask:
1588 case X86::BI__builtin_ia32_storedqusi128_mask:
1589 case X86::BI__builtin_ia32_storedquhi128_mask:
1590 case X86::BI__builtin_ia32_storedquqi128_mask:
1591 case X86::BI__builtin_ia32_storeupd128_mask:
1592 case X86::BI__builtin_ia32_storeups128_mask:
1593 case X86::BI__builtin_ia32_storedqudi256_mask:
1594 case X86::BI__builtin_ia32_storedqusi256_mask:
1595 case X86::BI__builtin_ia32_storedquhi256_mask:
1596 case X86::BI__builtin_ia32_storedquqi256_mask:
1597 case X86::BI__builtin_ia32_storeupd256_mask:
1598 case X86::BI__builtin_ia32_storeups256_mask:
1599 case X86::BI__builtin_ia32_storedqudi512_mask:
1600 case X86::BI__builtin_ia32_storedqusi512_mask:
1601 case X86::BI__builtin_ia32_storedquhi512_mask:
1602 case X86::BI__builtin_ia32_storedquqi512_mask:
1603 case X86::BI__builtin_ia32_storeupd512_mask:
1604 case X86::BI__builtin_ia32_storeups512_mask:
1605 case X86::BI__builtin_ia32_storesbf16128_mask:
1606 case X86::BI__builtin_ia32_storesh128_mask:
1607 case X86::BI__builtin_ia32_storess128_mask:
1608 case X86::BI__builtin_ia32_storesd128_mask:
1609 cgm.errorNYI(expr->getSourceRange(),
1610 std::string("unimplemented x86 builtin call: ") +
1611 getContext().BuiltinInfo.getName(ID: builtinID));
1612 return mlir::Value{};
1613 case X86::BI__builtin_ia32_cvtmask2b128:
1614 case X86::BI__builtin_ia32_cvtmask2b256:
1615 case X86::BI__builtin_ia32_cvtmask2b512:
1616 case X86::BI__builtin_ia32_cvtmask2w128:
1617 case X86::BI__builtin_ia32_cvtmask2w256:
1618 case X86::BI__builtin_ia32_cvtmask2w512:
1619 case X86::BI__builtin_ia32_cvtmask2d128:
1620 case X86::BI__builtin_ia32_cvtmask2d256:
1621 case X86::BI__builtin_ia32_cvtmask2d512:
1622 case X86::BI__builtin_ia32_cvtmask2q128:
1623 case X86::BI__builtin_ia32_cvtmask2q256:
1624 case X86::BI__builtin_ia32_cvtmask2q512:
1625 return emitX86SExtMask(builder&: this->getBuilder(), op: ops[0],
1626 dstTy: convertType(t: expr->getType()),
1627 loc: getLoc(srcLoc: expr->getExprLoc()));
1628 case X86::BI__builtin_ia32_cvtb2mask128:
1629 case X86::BI__builtin_ia32_cvtb2mask256:
1630 case X86::BI__builtin_ia32_cvtb2mask512:
1631 case X86::BI__builtin_ia32_cvtw2mask128:
1632 case X86::BI__builtin_ia32_cvtw2mask256:
1633 case X86::BI__builtin_ia32_cvtw2mask512:
1634 case X86::BI__builtin_ia32_cvtd2mask128:
1635 case X86::BI__builtin_ia32_cvtd2mask256:
1636 case X86::BI__builtin_ia32_cvtd2mask512:
1637 case X86::BI__builtin_ia32_cvtq2mask128:
1638 case X86::BI__builtin_ia32_cvtq2mask256:
1639 case X86::BI__builtin_ia32_cvtq2mask512:
1640 return emitX86ConvertToMask(cgf&: *this, builder&: this->getBuilder(), in: ops[0],
1641 loc: getLoc(srcLoc: expr->getExprLoc()));
1642 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
1643 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
1644 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
1645 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
1646 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
1647 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
1648 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
1649 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
1650 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
1651 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
1652 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
1653 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
1654 case X86::BI__builtin_ia32_vfmaddsh3_mask:
1655 case X86::BI__builtin_ia32_vfmaddss3_mask:
1656 case X86::BI__builtin_ia32_vfmaddsd3_mask:
1657 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
1658 case X86::BI__builtin_ia32_vfmaddss3_maskz:
1659 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
1660 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
1661 case X86::BI__builtin_ia32_vfmaddss3_mask3:
1662 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
1663 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
1664 case X86::BI__builtin_ia32_vfmsubss3_mask3:
1665 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
1666 case X86::BI__builtin_ia32_vfmaddph512_mask:
1667 case X86::BI__builtin_ia32_vfmaddph512_maskz:
1668 case X86::BI__builtin_ia32_vfmaddph512_mask3:
1669 case X86::BI__builtin_ia32_vfmaddps512_mask:
1670 case X86::BI__builtin_ia32_vfmaddps512_maskz:
1671 case X86::BI__builtin_ia32_vfmaddps512_mask3:
1672 case X86::BI__builtin_ia32_vfmsubps512_mask3:
1673 case X86::BI__builtin_ia32_vfmaddpd512_mask:
1674 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
1675 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
1676 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
1677 case X86::BI__builtin_ia32_vfmsubph512_mask3:
1678 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
1679 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
1680 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
1681 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
1682 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
1683 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
1684 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
1685 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
1686 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
1687 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
1688 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
1689 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
1690 case X86::BI__builtin_ia32_movdqa32store128_mask:
1691 case X86::BI__builtin_ia32_movdqa64store128_mask:
1692 case X86::BI__builtin_ia32_storeaps128_mask:
1693 case X86::BI__builtin_ia32_storeapd128_mask:
1694 case X86::BI__builtin_ia32_movdqa32store256_mask:
1695 case X86::BI__builtin_ia32_movdqa64store256_mask:
1696 case X86::BI__builtin_ia32_storeaps256_mask:
1697 case X86::BI__builtin_ia32_storeapd256_mask:
1698 case X86::BI__builtin_ia32_movdqa32store512_mask:
1699 case X86::BI__builtin_ia32_movdqa64store512_mask:
1700 case X86::BI__builtin_ia32_storeaps512_mask:
1701 case X86::BI__builtin_ia32_storeapd512_mask:
1702 cgm.errorNYI(expr->getSourceRange(),
1703 std::string("unimplemented X86 builtin call: ") +
1704 getContext().BuiltinInfo.getName(ID: builtinID));
1705 return {};
1706
1707 case X86::BI__builtin_ia32_loadups128_mask:
1708 case X86::BI__builtin_ia32_loadups256_mask:
1709 case X86::BI__builtin_ia32_loadups512_mask:
1710 case X86::BI__builtin_ia32_loadupd128_mask:
1711 case X86::BI__builtin_ia32_loadupd256_mask:
1712 case X86::BI__builtin_ia32_loadupd512_mask:
1713 case X86::BI__builtin_ia32_loaddquqi128_mask:
1714 case X86::BI__builtin_ia32_loaddquqi256_mask:
1715 case X86::BI__builtin_ia32_loaddquqi512_mask:
1716 case X86::BI__builtin_ia32_loaddquhi128_mask:
1717 case X86::BI__builtin_ia32_loaddquhi256_mask:
1718 case X86::BI__builtin_ia32_loaddquhi512_mask:
1719 case X86::BI__builtin_ia32_loaddqusi128_mask:
1720 case X86::BI__builtin_ia32_loaddqusi256_mask:
1721 case X86::BI__builtin_ia32_loaddqusi512_mask:
1722 case X86::BI__builtin_ia32_loaddqudi128_mask:
1723 case X86::BI__builtin_ia32_loaddqudi256_mask:
1724 case X86::BI__builtin_ia32_loaddqudi512_mask:
1725 case X86::BI__builtin_ia32_loadsbf16128_mask:
1726 case X86::BI__builtin_ia32_loadsh128_mask:
1727 case X86::BI__builtin_ia32_loadss128_mask:
1728 case X86::BI__builtin_ia32_loadsd128_mask:
1729 return emitX86MaskedLoad(builder, ops, alignment: llvm::Align(1),
1730 loc: getLoc(srcLoc: expr->getExprLoc()));
1731
1732 case X86::BI__builtin_ia32_loadaps128_mask:
1733 case X86::BI__builtin_ia32_loadaps256_mask:
1734 case X86::BI__builtin_ia32_loadaps512_mask:
1735 case X86::BI__builtin_ia32_loadapd128_mask:
1736 case X86::BI__builtin_ia32_loadapd256_mask:
1737 case X86::BI__builtin_ia32_loadapd512_mask:
1738 case X86::BI__builtin_ia32_movdqa32load128_mask:
1739 case X86::BI__builtin_ia32_movdqa32load256_mask:
1740 case X86::BI__builtin_ia32_movdqa32load512_mask:
1741 case X86::BI__builtin_ia32_movdqa64load128_mask:
1742 case X86::BI__builtin_ia32_movdqa64load256_mask:
1743 case X86::BI__builtin_ia32_movdqa64load512_mask:
1744 return emitX86MaskedLoad(
1745 builder, ops,
1746 alignment: getContext()
1747 .getTypeAlignInChars(T: expr->getArg(Arg: 1)->getType())
1748 .getAsAlign(),
1749 loc: getLoc(srcLoc: expr->getExprLoc()));
1750
1751 case X86::BI__builtin_ia32_expandloaddf128_mask:
1752 case X86::BI__builtin_ia32_expandloaddf256_mask:
1753 case X86::BI__builtin_ia32_expandloaddf512_mask:
1754 case X86::BI__builtin_ia32_expandloadsf128_mask:
1755 case X86::BI__builtin_ia32_expandloadsf256_mask:
1756 case X86::BI__builtin_ia32_expandloadsf512_mask:
1757 case X86::BI__builtin_ia32_expandloaddi128_mask:
1758 case X86::BI__builtin_ia32_expandloaddi256_mask:
1759 case X86::BI__builtin_ia32_expandloaddi512_mask:
1760 case X86::BI__builtin_ia32_expandloadsi128_mask:
1761 case X86::BI__builtin_ia32_expandloadsi256_mask:
1762 case X86::BI__builtin_ia32_expandloadsi512_mask:
1763 case X86::BI__builtin_ia32_expandloadhi128_mask:
1764 case X86::BI__builtin_ia32_expandloadhi256_mask:
1765 case X86::BI__builtin_ia32_expandloadhi512_mask:
1766 case X86::BI__builtin_ia32_expandloadqi128_mask:
1767 case X86::BI__builtin_ia32_expandloadqi256_mask:
1768 case X86::BI__builtin_ia32_expandloadqi512_mask: {
1769 cgm.errorNYI(expr->getSourceRange(),
1770 std::string("unimplemented X86 builtin call: ") +
1771 getContext().BuiltinInfo.getName(ID: builtinID));
1772 return {};
1773 }
1774 case X86::BI__builtin_ia32_compressstoredf128_mask:
1775 case X86::BI__builtin_ia32_compressstoredf256_mask:
1776 case X86::BI__builtin_ia32_compressstoredf512_mask:
1777 case X86::BI__builtin_ia32_compressstoresf128_mask:
1778 case X86::BI__builtin_ia32_compressstoresf256_mask:
1779 case X86::BI__builtin_ia32_compressstoresf512_mask:
1780 case X86::BI__builtin_ia32_compressstoredi128_mask:
1781 case X86::BI__builtin_ia32_compressstoredi256_mask:
1782 case X86::BI__builtin_ia32_compressstoredi512_mask:
1783 case X86::BI__builtin_ia32_compressstoresi128_mask:
1784 case X86::BI__builtin_ia32_compressstoresi256_mask:
1785 case X86::BI__builtin_ia32_compressstoresi512_mask:
1786 case X86::BI__builtin_ia32_compressstorehi128_mask:
1787 case X86::BI__builtin_ia32_compressstorehi256_mask:
1788 case X86::BI__builtin_ia32_compressstorehi512_mask:
1789 case X86::BI__builtin_ia32_compressstoreqi128_mask:
1790 case X86::BI__builtin_ia32_compressstoreqi256_mask:
1791 case X86::BI__builtin_ia32_compressstoreqi512_mask:
1792 return emitX86CompressStore(builder, loc: getLoc(srcLoc: expr->getExprLoc()), ops);
1793 case X86::BI__builtin_ia32_expanddf128_mask:
1794 case X86::BI__builtin_ia32_expanddf256_mask:
1795 case X86::BI__builtin_ia32_expanddf512_mask:
1796 case X86::BI__builtin_ia32_expandsf128_mask:
1797 case X86::BI__builtin_ia32_expandsf256_mask:
1798 case X86::BI__builtin_ia32_expandsf512_mask:
1799 case X86::BI__builtin_ia32_expanddi128_mask:
1800 case X86::BI__builtin_ia32_expanddi256_mask:
1801 case X86::BI__builtin_ia32_expanddi512_mask:
1802 case X86::BI__builtin_ia32_expandsi128_mask:
1803 case X86::BI__builtin_ia32_expandsi256_mask:
1804 case X86::BI__builtin_ia32_expandsi512_mask:
1805 case X86::BI__builtin_ia32_expandhi128_mask:
1806 case X86::BI__builtin_ia32_expandhi256_mask:
1807 case X86::BI__builtin_ia32_expandhi512_mask:
1808 case X86::BI__builtin_ia32_expandqi128_mask:
1809 case X86::BI__builtin_ia32_expandqi256_mask:
1810 case X86::BI__builtin_ia32_expandqi512_mask: {
1811 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
1812 return emitX86CompressExpand(builder, loc, source: ops[0], mask: ops[1], inputVector: ops[2],
1813 id: "x86.avx512.mask.expand");
1814 }
1815 case X86::BI__builtin_ia32_compressdf128_mask:
1816 case X86::BI__builtin_ia32_compressdf256_mask:
1817 case X86::BI__builtin_ia32_compressdf512_mask:
1818 case X86::BI__builtin_ia32_compresssf128_mask:
1819 case X86::BI__builtin_ia32_compresssf256_mask:
1820 case X86::BI__builtin_ia32_compresssf512_mask:
1821 case X86::BI__builtin_ia32_compressdi128_mask:
1822 case X86::BI__builtin_ia32_compressdi256_mask:
1823 case X86::BI__builtin_ia32_compressdi512_mask:
1824 case X86::BI__builtin_ia32_compresssi128_mask:
1825 case X86::BI__builtin_ia32_compresssi256_mask:
1826 case X86::BI__builtin_ia32_compresssi512_mask:
1827 case X86::BI__builtin_ia32_compresshi128_mask:
1828 case X86::BI__builtin_ia32_compresshi256_mask:
1829 case X86::BI__builtin_ia32_compresshi512_mask:
1830 case X86::BI__builtin_ia32_compressqi128_mask:
1831 case X86::BI__builtin_ia32_compressqi256_mask:
1832 case X86::BI__builtin_ia32_compressqi512_mask: {
1833 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
1834 return emitX86CompressExpand(builder, loc, source: ops[0], mask: ops[1], inputVector: ops[2],
1835 id: "x86.avx512.mask.compress");
1836 }
1837 case X86::BI__builtin_ia32_gather3div2df:
1838 case X86::BI__builtin_ia32_gather3div2di:
1839 case X86::BI__builtin_ia32_gather3div4df:
1840 case X86::BI__builtin_ia32_gather3div4di:
1841 case X86::BI__builtin_ia32_gather3div4sf:
1842 case X86::BI__builtin_ia32_gather3div4si:
1843 case X86::BI__builtin_ia32_gather3div8sf:
1844 case X86::BI__builtin_ia32_gather3div8si:
1845 case X86::BI__builtin_ia32_gather3siv2df:
1846 case X86::BI__builtin_ia32_gather3siv2di:
1847 case X86::BI__builtin_ia32_gather3siv4df:
1848 case X86::BI__builtin_ia32_gather3siv4di:
1849 case X86::BI__builtin_ia32_gather3siv4sf:
1850 case X86::BI__builtin_ia32_gather3siv4si:
1851 case X86::BI__builtin_ia32_gather3siv8sf:
1852 case X86::BI__builtin_ia32_gather3siv8si:
1853 case X86::BI__builtin_ia32_gathersiv8df:
1854 case X86::BI__builtin_ia32_gathersiv16sf:
1855 case X86::BI__builtin_ia32_gatherdiv8df:
1856 case X86::BI__builtin_ia32_gatherdiv16sf:
1857 case X86::BI__builtin_ia32_gathersiv8di:
1858 case X86::BI__builtin_ia32_gathersiv16si:
1859 case X86::BI__builtin_ia32_gatherdiv8di:
1860 case X86::BI__builtin_ia32_gatherdiv16si: {
1861 StringRef intrinsicName;
1862 switch (builtinID) {
1863 default:
1864 llvm_unreachable("Unexpected builtin");
1865 case X86::BI__builtin_ia32_gather3div2df:
1866 intrinsicName = "x86.avx512.mask.gather3div2.df";
1867 break;
1868 case X86::BI__builtin_ia32_gather3div2di:
1869 intrinsicName = "x86.avx512.mask.gather3div2.di";
1870 break;
1871 case X86::BI__builtin_ia32_gather3div4df:
1872 intrinsicName = "x86.avx512.mask.gather3div4.df";
1873 break;
1874 case X86::BI__builtin_ia32_gather3div4di:
1875 intrinsicName = "x86.avx512.mask.gather3div4.di";
1876 break;
1877 case X86::BI__builtin_ia32_gather3div4sf:
1878 intrinsicName = "x86.avx512.mask.gather3div4.sf";
1879 break;
1880 case X86::BI__builtin_ia32_gather3div4si:
1881 intrinsicName = "x86.avx512.mask.gather3div4.si";
1882 break;
1883 case X86::BI__builtin_ia32_gather3div8sf:
1884 intrinsicName = "x86.avx512.mask.gather3div8.sf";
1885 break;
1886 case X86::BI__builtin_ia32_gather3div8si:
1887 intrinsicName = "x86.avx512.mask.gather3div8.si";
1888 break;
1889 case X86::BI__builtin_ia32_gather3siv2df:
1890 intrinsicName = "x86.avx512.mask.gather3siv2.df";
1891 break;
1892 case X86::BI__builtin_ia32_gather3siv2di:
1893 intrinsicName = "x86.avx512.mask.gather3siv2.di";
1894 break;
1895 case X86::BI__builtin_ia32_gather3siv4df:
1896 intrinsicName = "x86.avx512.mask.gather3siv4.df";
1897 break;
1898 case X86::BI__builtin_ia32_gather3siv4di:
1899 intrinsicName = "x86.avx512.mask.gather3siv4.di";
1900 break;
1901 case X86::BI__builtin_ia32_gather3siv4sf:
1902 intrinsicName = "x86.avx512.mask.gather3siv4.sf";
1903 break;
1904 case X86::BI__builtin_ia32_gather3siv4si:
1905 intrinsicName = "x86.avx512.mask.gather3siv4.si";
1906 break;
1907 case X86::BI__builtin_ia32_gather3siv8sf:
1908 intrinsicName = "x86.avx512.mask.gather3siv8.sf";
1909 break;
1910 case X86::BI__builtin_ia32_gather3siv8si:
1911 intrinsicName = "x86.avx512.mask.gather3siv8.si";
1912 break;
1913 case X86::BI__builtin_ia32_gathersiv8df:
1914 intrinsicName = "x86.avx512.mask.gather.dpd.512";
1915 break;
1916 case X86::BI__builtin_ia32_gathersiv16sf:
1917 intrinsicName = "x86.avx512.mask.gather.dps.512";
1918 break;
1919 case X86::BI__builtin_ia32_gatherdiv8df:
1920 intrinsicName = "x86.avx512.mask.gather.qpd.512";
1921 break;
1922 case X86::BI__builtin_ia32_gatherdiv16sf:
1923 intrinsicName = "x86.avx512.mask.gather.qps.512";
1924 break;
1925 case X86::BI__builtin_ia32_gathersiv8di:
1926 intrinsicName = "x86.avx512.mask.gather.dpq.512";
1927 break;
1928 case X86::BI__builtin_ia32_gathersiv16si:
1929 intrinsicName = "x86.avx512.mask.gather.dpi.512";
1930 break;
1931 case X86::BI__builtin_ia32_gatherdiv8di:
1932 intrinsicName = "x86.avx512.mask.gather.qpq.512";
1933 break;
1934 case X86::BI__builtin_ia32_gatherdiv16si:
1935 intrinsicName = "x86.avx512.mask.gather.qpi.512";
1936 break;
1937 }
1938
1939 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
1940 unsigned minElts =
1941 std::min(a: cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize(),
1942 b: cast<cir::VectorType>(Val: ops[2].getType()).getFixedSize());
1943 ops[3] = getMaskVecValue(builder, loc, mask: ops[3], numElems: minElts);
1944 return builder.emitIntrinsicCallOp(loc, str: intrinsicName,
1945 resTy: convertType(t: expr->getType()), op&: ops);
1946 }
1947 case X86::BI__builtin_ia32_scattersiv8df:
1948 case X86::BI__builtin_ia32_scattersiv16sf:
1949 case X86::BI__builtin_ia32_scatterdiv8df:
1950 case X86::BI__builtin_ia32_scatterdiv16sf:
1951 case X86::BI__builtin_ia32_scattersiv8di:
1952 case X86::BI__builtin_ia32_scattersiv16si:
1953 case X86::BI__builtin_ia32_scatterdiv8di:
1954 case X86::BI__builtin_ia32_scatterdiv16si:
1955 case X86::BI__builtin_ia32_scatterdiv2df:
1956 case X86::BI__builtin_ia32_scatterdiv2di:
1957 case X86::BI__builtin_ia32_scatterdiv4df:
1958 case X86::BI__builtin_ia32_scatterdiv4di:
1959 case X86::BI__builtin_ia32_scatterdiv4sf:
1960 case X86::BI__builtin_ia32_scatterdiv4si:
1961 case X86::BI__builtin_ia32_scatterdiv8sf:
1962 case X86::BI__builtin_ia32_scatterdiv8si:
1963 case X86::BI__builtin_ia32_scattersiv2df:
1964 case X86::BI__builtin_ia32_scattersiv2di:
1965 case X86::BI__builtin_ia32_scattersiv4df:
1966 case X86::BI__builtin_ia32_scattersiv4di:
1967 case X86::BI__builtin_ia32_scattersiv4sf:
1968 case X86::BI__builtin_ia32_scattersiv4si:
1969 case X86::BI__builtin_ia32_scattersiv8sf:
1970 case X86::BI__builtin_ia32_scattersiv8si: {
1971 llvm::StringRef intrinsicName;
1972 switch (builtinID) {
1973 default:
1974 llvm_unreachable("Unexpected builtin");
1975 case X86::BI__builtin_ia32_scattersiv8df:
1976 intrinsicName = "x86.avx512.mask.scatter.dpd.512";
1977 break;
1978 case X86::BI__builtin_ia32_scattersiv16sf:
1979 intrinsicName = "x86.avx512.mask.scatter.dps.512";
1980 break;
1981 case X86::BI__builtin_ia32_scatterdiv8df:
1982 intrinsicName = "x86.avx512.mask.scatter.qpd.512";
1983 break;
1984 case X86::BI__builtin_ia32_scatterdiv16sf:
1985 intrinsicName = "x86.avx512.mask.scatter.qps.512";
1986 break;
1987 case X86::BI__builtin_ia32_scattersiv8di:
1988 intrinsicName = "x86.avx512.mask.scatter.dpq.512";
1989 break;
1990 case X86::BI__builtin_ia32_scattersiv16si:
1991 intrinsicName = "x86.avx512.mask.scatter.dpi.512";
1992 break;
1993 case X86::BI__builtin_ia32_scatterdiv8di:
1994 intrinsicName = "x86.avx512.mask.scatter.qpq.512";
1995 break;
1996 case X86::BI__builtin_ia32_scatterdiv16si:
1997 intrinsicName = "x86.avx512.mask.scatter.qpi.512";
1998 break;
1999 case X86::BI__builtin_ia32_scatterdiv2df:
2000 intrinsicName = "x86.avx512.mask.scatterdiv2.df";
2001 break;
2002 case X86::BI__builtin_ia32_scatterdiv2di:
2003 intrinsicName = "x86.avx512.mask.scatterdiv2.di";
2004 break;
2005 case X86::BI__builtin_ia32_scatterdiv4df:
2006 intrinsicName = "x86.avx512.mask.scatterdiv4.df";
2007 break;
2008 case X86::BI__builtin_ia32_scatterdiv4di:
2009 intrinsicName = "x86.avx512.mask.scatterdiv4.di";
2010 break;
2011 case X86::BI__builtin_ia32_scatterdiv4sf:
2012 intrinsicName = "x86.avx512.mask.scatterdiv4.sf";
2013 break;
2014 case X86::BI__builtin_ia32_scatterdiv4si:
2015 intrinsicName = "x86.avx512.mask.scatterdiv4.si";
2016 break;
2017 case X86::BI__builtin_ia32_scatterdiv8sf:
2018 intrinsicName = "x86.avx512.mask.scatterdiv8.sf";
2019 break;
2020 case X86::BI__builtin_ia32_scatterdiv8si:
2021 intrinsicName = "x86.avx512.mask.scatterdiv8.si";
2022 break;
2023 case X86::BI__builtin_ia32_scattersiv2df:
2024 intrinsicName = "x86.avx512.mask.scattersiv2.df";
2025 break;
2026 case X86::BI__builtin_ia32_scattersiv2di:
2027 intrinsicName = "x86.avx512.mask.scattersiv2.di";
2028 break;
2029 case X86::BI__builtin_ia32_scattersiv4df:
2030 intrinsicName = "x86.avx512.mask.scattersiv4.df";
2031 break;
2032 case X86::BI__builtin_ia32_scattersiv4di:
2033 intrinsicName = "x86.avx512.mask.scattersiv4.di";
2034 break;
2035 case X86::BI__builtin_ia32_scattersiv4sf:
2036 intrinsicName = "x86.avx512.mask.scattersiv4.sf";
2037 break;
2038 case X86::BI__builtin_ia32_scattersiv4si:
2039 intrinsicName = "x86.avx512.mask.scattersiv4.si";
2040 break;
2041 case X86::BI__builtin_ia32_scattersiv8sf:
2042 intrinsicName = "x86.avx512.mask.scattersiv8.sf";
2043 break;
2044 case X86::BI__builtin_ia32_scattersiv8si:
2045 intrinsicName = "x86.avx512.mask.scattersiv8.si";
2046 break;
2047 }
2048
2049 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2050 unsigned minElts =
2051 std::min(a: cast<cir::VectorType>(Val: ops[2].getType()).getFixedSize(),
2052 b: cast<cir::VectorType>(Val: ops[3].getType()).getFixedSize());
2053 ops[1] = getMaskVecValue(builder, loc, mask: ops[1], numElems: minElts);
2054
2055 return builder.emitIntrinsicCallOp(loc, str: intrinsicName,
2056 resTy: convertType(t: expr->getType()), op&: ops);
2057 }
2058 case X86::BI__builtin_ia32_vextractf128_pd256:
2059 case X86::BI__builtin_ia32_vextractf128_ps256:
2060 case X86::BI__builtin_ia32_vextractf128_si256:
2061 case X86::BI__builtin_ia32_extract128i256:
2062 case X86::BI__builtin_ia32_extractf64x4_mask:
2063 case X86::BI__builtin_ia32_extractf32x4_mask:
2064 case X86::BI__builtin_ia32_extracti64x4_mask:
2065 case X86::BI__builtin_ia32_extracti32x4_mask:
2066 case X86::BI__builtin_ia32_extractf32x8_mask:
2067 case X86::BI__builtin_ia32_extracti32x8_mask:
2068 case X86::BI__builtin_ia32_extractf32x4_256_mask:
2069 case X86::BI__builtin_ia32_extracti32x4_256_mask:
2070 case X86::BI__builtin_ia32_extractf64x2_256_mask:
2071 case X86::BI__builtin_ia32_extracti64x2_256_mask:
2072 case X86::BI__builtin_ia32_extractf64x2_512_mask:
2073 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
2074 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2075 cir::VectorType dstTy = cast<cir::VectorType>(Val: convertType(t: expr->getType()));
2076 unsigned numElts = dstTy.getFixedSize();
2077 unsigned srcNumElts =
2078 cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize();
2079 unsigned subVectors = srcNumElts / numElts;
2080 assert(llvm::isPowerOf2_32(subVectors) && "Expected power of 2 subvectors");
2081 unsigned index =
2082 ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue();
2083
2084 index &= subVectors - 1; // Remove any extra bits.
2085 index *= numElts;
2086
2087 int64_t indices[16];
2088 std::iota(first: indices, last: indices + numElts, value: index);
2089
2090 mlir::Value poison =
2091 builder.getConstant(loc, attr: cir::PoisonAttr::get(type: ops[0].getType()));
2092 mlir::Value res = builder.createVecShuffle(loc, vec1: ops[0], vec2: poison,
2093 mask: ArrayRef(indices, numElts));
2094 if (ops.size() == 4)
2095 res = emitX86Select(builder, loc, mask: ops[3], op0: res, op1: ops[2]);
2096
2097 return res;
2098 }
2099 case X86::BI__builtin_ia32_vinsertf128_pd256:
2100 case X86::BI__builtin_ia32_vinsertf128_ps256:
2101 case X86::BI__builtin_ia32_vinsertf128_si256:
2102 case X86::BI__builtin_ia32_insert128i256:
2103 case X86::BI__builtin_ia32_insertf64x4:
2104 case X86::BI__builtin_ia32_insertf32x4:
2105 case X86::BI__builtin_ia32_inserti64x4:
2106 case X86::BI__builtin_ia32_inserti32x4:
2107 case X86::BI__builtin_ia32_insertf32x8:
2108 case X86::BI__builtin_ia32_inserti32x8:
2109 case X86::BI__builtin_ia32_insertf32x4_256:
2110 case X86::BI__builtin_ia32_inserti32x4_256:
2111 case X86::BI__builtin_ia32_insertf64x2_256:
2112 case X86::BI__builtin_ia32_inserti64x2_256:
2113 case X86::BI__builtin_ia32_insertf64x2_512:
2114 case X86::BI__builtin_ia32_inserti64x2_512: {
2115 unsigned dstNumElts =
2116 cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize();
2117 unsigned srcNumElts =
2118 cast<cir::VectorType>(Val: ops[1].getType()).getFixedSize();
2119 unsigned subVectors = dstNumElts / srcNumElts;
2120 assert(llvm::isPowerOf2_32(subVectors) && "Expected power of 2 subvectors");
2121 assert(dstNumElts <= 16);
2122
2123 uint64_t index = getZExtIntValueFromConstOp(val: ops[2]);
2124 index &= subVectors - 1; // Remove any extra bits.
2125 index *= srcNumElts;
2126
2127 llvm::SmallVector<int64_t, 16> mask(dstNumElts);
2128 for (unsigned i = 0; i != dstNumElts; ++i)
2129 mask[i] = (i >= srcNumElts) ? srcNumElts + (i % srcNumElts) : i;
2130
2131 mlir::Value op1 =
2132 builder.createVecShuffle(loc: getLoc(srcLoc: expr->getExprLoc()), vec1: ops[1], mask);
2133
2134 for (unsigned i = 0; i != dstNumElts; ++i) {
2135 if (i >= index && i < (index + srcNumElts))
2136 mask[i] = (i - index) + dstNumElts;
2137 else
2138 mask[i] = i;
2139 }
2140
2141 return builder.createVecShuffle(loc: getLoc(srcLoc: expr->getExprLoc()), vec1: ops[0], vec2: op1,
2142 mask);
2143 }
2144 case X86::BI__builtin_ia32_pmovqd512_mask:
2145 case X86::BI__builtin_ia32_pmovwb512_mask: {
2146 mlir::Value Res =
2147 builder.createIntCast(src: ops[0], newTy: cast<cir::VectorType>(Val: ops[1].getType()));
2148 return emitX86Select(builder, loc: getLoc(srcLoc: expr->getExprLoc()), mask: ops[2], op0: Res,
2149 op1: ops[1]);
2150 }
2151 case X86::BI__builtin_ia32_pmovdb512_mask:
2152 case X86::BI__builtin_ia32_pmovdw512_mask:
2153 case X86::BI__builtin_ia32_pmovqw512_mask: {
2154 // These only require AVX512F, but a byte/word select would need AVX512BW,
2155 // so use the masked intrinsic unless the mask is all ones.
2156 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2157 auto c = mlir::dyn_cast_or_null<cir::ConstantOp>(Val: ops[2].getDefiningOp());
2158 if (c && c.isAllOnesValue())
2159 return builder.createIntCast(src: ops[0],
2160 newTy: cast<cir::VectorType>(Val: ops[1].getType()));
2161
2162 llvm::StringRef intrinsicName;
2163 switch (builtinID) {
2164 default:
2165 llvm_unreachable("Unsupported intrinsic!");
2166 case X86::BI__builtin_ia32_pmovdb512_mask:
2167 intrinsicName = "x86.avx512.mask.pmov.db.512";
2168 break;
2169 case X86::BI__builtin_ia32_pmovdw512_mask:
2170 intrinsicName = "x86.avx512.mask.pmov.dw.512";
2171 break;
2172 case X86::BI__builtin_ia32_pmovqw512_mask:
2173 intrinsicName = "x86.avx512.mask.pmov.qw.512";
2174 break;
2175 }
2176 return builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: ops[1].getType(),
2177 op&: ops);
2178 }
2179 case X86::BI__builtin_ia32_pblendw128:
2180 case X86::BI__builtin_ia32_blendpd:
2181 case X86::BI__builtin_ia32_blendps:
2182 case X86::BI__builtin_ia32_blendpd256:
2183 case X86::BI__builtin_ia32_blendps256:
2184 case X86::BI__builtin_ia32_pblendw256:
2185 case X86::BI__builtin_ia32_pblendd128:
2186 case X86::BI__builtin_ia32_pblendd256: {
2187 uint32_t imm = getZExtIntValueFromConstOp(val: ops[2]);
2188 unsigned numElts = cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize();
2189
2190 llvm::SmallVector<mlir::Attribute, 16> indices;
2191 // If there are more than 8 elements, the immediate is used twice so make
2192 // sure we handle that.
2193 mlir::Type i32Ty = builder.getSInt32Ty();
2194 for (unsigned i = 0; i != numElts; ++i)
2195 indices.push_back(
2196 Elt: cir::IntAttr::get(type: i32Ty, value: ((imm >> (i % 8)) & 0x1) ? numElts + i : i));
2197
2198 return builder.createVecShuffle(loc: getLoc(srcLoc: expr->getExprLoc()), vec1: ops[0], vec2: ops[1],
2199 maskAttrs: indices);
2200 }
2201 case X86::BI__builtin_ia32_pshuflw:
2202 case X86::BI__builtin_ia32_pshuflw256:
2203 case X86::BI__builtin_ia32_pshuflw512:
2204 return emitPshufWord(builder, vec: ops[0], immediate: ops[1], loc: getLoc(srcLoc: expr->getExprLoc()),
2205 isLow: true);
2206 case X86::BI__builtin_ia32_pshufhw:
2207 case X86::BI__builtin_ia32_pshufhw256:
2208 case X86::BI__builtin_ia32_pshufhw512:
2209 return emitPshufWord(builder, vec: ops[0], immediate: ops[1], loc: getLoc(srcLoc: expr->getExprLoc()),
2210 isLow: false);
2211 case X86::BI__builtin_ia32_pshufd:
2212 case X86::BI__builtin_ia32_pshufd256:
2213 case X86::BI__builtin_ia32_pshufd512:
2214 case X86::BI__builtin_ia32_vpermilpd:
2215 case X86::BI__builtin_ia32_vpermilps:
2216 case X86::BI__builtin_ia32_vpermilpd256:
2217 case X86::BI__builtin_ia32_vpermilps256:
2218 case X86::BI__builtin_ia32_vpermilpd512:
2219 case X86::BI__builtin_ia32_vpermilps512: {
2220 const uint32_t imm = getSExtIntValueFromConstOp(val: ops[1]);
2221
2222 llvm::SmallVector<int64_t, 16> mask(16);
2223 computeFullLaneShuffleMask(cgf&: *this, vec: ops[0], imm, isShufP: false, outIndices&: mask);
2224
2225 return builder.createVecShuffle(loc: getLoc(srcLoc: expr->getExprLoc()), vec1: ops[0], mask);
2226 }
2227 case X86::BI__builtin_ia32_shufpd:
2228 case X86::BI__builtin_ia32_shufpd256:
2229 case X86::BI__builtin_ia32_shufpd512:
2230 case X86::BI__builtin_ia32_shufps:
2231 case X86::BI__builtin_ia32_shufps256:
2232 case X86::BI__builtin_ia32_shufps512: {
2233 const uint32_t imm = getZExtIntValueFromConstOp(val: ops[2]);
2234
2235 llvm::SmallVector<int64_t, 16> mask(16);
2236 computeFullLaneShuffleMask(cgf&: *this, vec: ops[0], imm, isShufP: true, outIndices&: mask);
2237
2238 return builder.createVecShuffle(loc: getLoc(srcLoc: expr->getExprLoc()), vec1: ops[0], vec2: ops[1],
2239 mask);
2240 }
2241 case X86::BI__builtin_ia32_permdi256:
2242 case X86::BI__builtin_ia32_permdf256:
2243 case X86::BI__builtin_ia32_permdi512:
2244 case X86::BI__builtin_ia32_permdf512: {
2245 unsigned imm =
2246 ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue();
2247 unsigned numElts = cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize();
2248
2249 // These intrinsics operate on 256-bit lanes of four 64-bit elements.
2250 int64_t Indices[8];
2251
2252 for (unsigned l = 0; l != numElts; l += 4)
2253 for (unsigned i = 0; i != 4; ++i)
2254 Indices[l + i] = l + ((imm >> (2 * i)) & 0x3);
2255
2256 return builder.createVecShuffle(loc: getLoc(srcLoc: expr->getExprLoc()), vec1: ops[0],
2257 mask: ArrayRef(Indices, numElts));
2258 }
2259 case X86::BI__builtin_ia32_palignr128:
2260 case X86::BI__builtin_ia32_palignr256:
2261 case X86::BI__builtin_ia32_palignr512: {
2262 uint32_t shiftVal = getZExtIntValueFromConstOp(val: ops[2]) & 0xff;
2263
2264 unsigned numElts = cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize();
2265 assert(numElts % 16 == 0);
2266
2267 // If palignr is shifting the pair of vectors more than the size of two
2268 // lanes, emit zero.
2269 if (shiftVal >= 32)
2270 return builder.getNullValue(ty: convertType(t: expr->getType()),
2271 loc: getLoc(srcLoc: expr->getExprLoc()));
2272
2273 // If palignr is shifting the pair of input vectors more than one lane,
2274 // but less than two lanes, convert to shifting in zeroes.
2275 if (shiftVal > 16) {
2276 shiftVal -= 16;
2277 ops[1] = ops[0];
2278 ops[0] =
2279 builder.getNullValue(ty: ops[0].getType(), loc: getLoc(srcLoc: expr->getExprLoc()));
2280 }
2281
2282 int64_t indices[64];
2283 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2284 for (unsigned l = 0; l != numElts; l += 16) {
2285 for (unsigned i = 0; i != 16; ++i) {
2286 uint32_t idx = shiftVal + i;
2287 if (idx >= 16)
2288 idx += numElts - 16; // End of lane, switch operand.
2289 indices[l + i] = l + idx;
2290 }
2291 }
2292
2293 return builder.createVecShuffle(loc: getLoc(srcLoc: expr->getExprLoc()), vec1: ops[1], vec2: ops[0],
2294 mask: ArrayRef(indices, numElts));
2295 }
2296 case X86::BI__builtin_ia32_alignd128:
2297 case X86::BI__builtin_ia32_alignd256:
2298 case X86::BI__builtin_ia32_alignd512:
2299 case X86::BI__builtin_ia32_alignq128:
2300 case X86::BI__builtin_ia32_alignq256:
2301 case X86::BI__builtin_ia32_alignq512: {
2302 unsigned numElts = cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize();
2303 unsigned shiftVal =
2304 ops[2].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue() &
2305 0xff;
2306
2307 // Mask the shift amount to width of a vector.
2308 shiftVal &= numElts - 1;
2309
2310 SmallVector<mlir::Attribute, 16> indices;
2311 mlir::Type i32Ty = builder.getSInt32Ty();
2312 for (unsigned i = 0; i != numElts; ++i)
2313 indices.push_back(Elt: cir::IntAttr::get(type: i32Ty, value: i + shiftVal));
2314
2315 return builder.createVecShuffle(loc: getLoc(srcLoc: expr->getExprLoc()), vec1: ops[0], vec2: ops[1],
2316 maskAttrs: indices);
2317 }
2318 case X86::BI__builtin_ia32_shuf_f32x4_256:
2319 case X86::BI__builtin_ia32_shuf_f64x2_256:
2320 case X86::BI__builtin_ia32_shuf_i32x4_256:
2321 case X86::BI__builtin_ia32_shuf_i64x2_256:
2322 case X86::BI__builtin_ia32_shuf_f32x4:
2323 case X86::BI__builtin_ia32_shuf_f64x2:
2324 case X86::BI__builtin_ia32_shuf_i32x4:
2325 case X86::BI__builtin_ia32_shuf_i64x2: {
2326 mlir::Value src1 = ops[0];
2327 mlir::Value src2 = ops[1];
2328
2329 unsigned imm =
2330 ops[2].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue();
2331
2332 unsigned numElems = cast<cir::VectorType>(Val: src1.getType()).getFixedSize();
2333 unsigned totalBits = getContext().getTypeSize(T: expr->getArg(Arg: 0)->getType());
2334 unsigned numLanes = totalBits == 512 ? 4 : 2;
2335 unsigned numElemsPerLane = numElems / numLanes;
2336
2337 SmallVector<mlir::Attribute, 16> indices;
2338 mlir::Type i32Ty = builder.getSInt32Ty();
2339
2340 for (unsigned l = 0; l != numElems; l += numElemsPerLane) {
2341 unsigned index = (imm % numLanes) * numElemsPerLane;
2342 imm /= numLanes;
2343 if (l >= (numElems / 2))
2344 index += numElems;
2345 for (unsigned i = 0; i != numElemsPerLane; ++i) {
2346 indices.push_back(Elt: cir::IntAttr::get(type: i32Ty, value: index + i));
2347 }
2348 }
2349
2350 return builder.createVecShuffle(loc: getLoc(srcLoc: expr->getExprLoc()), vec1: src1, vec2: src2,
2351 maskAttrs: indices);
2352 }
2353 case X86::BI__builtin_ia32_vperm2f128_pd256:
2354 case X86::BI__builtin_ia32_vperm2f128_ps256:
2355 case X86::BI__builtin_ia32_vperm2f128_si256:
2356 case X86::BI__builtin_ia32_permti256:
2357 return emitX86VPerm2f128(builder, loc: getLoc(srcLoc: expr->getExprLoc()), ops);
2358 case X86::BI__builtin_ia32_pslldqi128_byteshift:
2359 case X86::BI__builtin_ia32_pslldqi256_byteshift:
2360 case X86::BI__builtin_ia32_pslldqi512_byteshift:
2361 return emitX86PackedByteShift(builder, builtinID,
2362 loc: getLoc(srcLoc: expr->getExprLoc()), ops,
2363 /**isLeftShift=*/isLeftShift: true);
2364 case X86::BI__builtin_ia32_psrldqi128_byteshift:
2365 case X86::BI__builtin_ia32_psrldqi256_byteshift:
2366 case X86::BI__builtin_ia32_psrldqi512_byteshift:
2367 return emitX86PackedByteShift(builder, builtinID,
2368 loc: getLoc(srcLoc: expr->getExprLoc()), ops,
2369 /**isLeftShift=*/isLeftShift: false);
2370 case X86::BI__builtin_ia32_kshiftliqi:
2371 case X86::BI__builtin_ia32_kshiftlihi:
2372 case X86::BI__builtin_ia32_kshiftlisi:
2373 case X86::BI__builtin_ia32_kshiftlidi: {
2374 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2375 unsigned shiftVal =
2376 ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue() &
2377 0xff;
2378 unsigned numElems = cast<cir::IntType>(Val: ops[0].getType()).getWidth();
2379
2380 if (shiftVal >= numElems)
2381 return builder.getNullValue(ty: ops[0].getType(), loc);
2382
2383 mlir::Value in = getMaskVecValue(builder, loc, mask: ops[0], numElems);
2384
2385 SmallVector<mlir::Attribute, 64> indices;
2386 mlir::Type i32Ty = builder.getSInt32Ty();
2387 for (auto i : llvm::seq<unsigned>(Begin: 0, End: numElems))
2388 indices.push_back(Elt: cir::IntAttr::get(type: i32Ty, value: numElems + i - shiftVal));
2389
2390 mlir::Value zero = builder.getNullValue(ty: in.getType(), loc);
2391 mlir::Value sv = builder.createVecShuffle(loc, vec1: zero, vec2: in, maskAttrs: indices);
2392 return builder.createBitcast(src: sv, newTy: ops[0].getType());
2393 }
2394 case X86::BI__builtin_ia32_kshiftriqi:
2395 case X86::BI__builtin_ia32_kshiftrihi:
2396 case X86::BI__builtin_ia32_kshiftrisi:
2397 case X86::BI__builtin_ia32_kshiftridi: {
2398 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2399 unsigned shiftVal =
2400 ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue() &
2401 0xff;
2402 unsigned numElems = cast<cir::IntType>(Val: ops[0].getType()).getWidth();
2403
2404 if (shiftVal >= numElems)
2405 return builder.getNullValue(ty: ops[0].getType(), loc);
2406
2407 mlir::Value in = getMaskVecValue(builder, loc, mask: ops[0], numElems);
2408
2409 SmallVector<mlir::Attribute, 64> indices;
2410 mlir::Type i32Ty = builder.getSInt32Ty();
2411 for (auto i : llvm::seq<unsigned>(Begin: 0, End: numElems))
2412 indices.push_back(Elt: cir::IntAttr::get(type: i32Ty, value: i + shiftVal));
2413
2414 mlir::Value zero = builder.getNullValue(ty: in.getType(), loc);
2415 mlir::Value sv = builder.createVecShuffle(loc, vec1: in, vec2: zero, maskAttrs: indices);
2416 return builder.createBitcast(src: sv, newTy: ops[0].getType());
2417 }
2418 case X86::BI__builtin_ia32_movnti:
2419 case X86::BI__builtin_ia32_movnti64:
2420 case X86::BI__builtin_ia32_movntsd:
2421 case X86::BI__builtin_ia32_movntss: {
2422 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2423
2424 Address dest = Address{ops[0], CharUnits::One()};
2425 mlir::Value src = ops[1];
2426
2427 if (builtinID == X86::BI__builtin_ia32_movntsd ||
2428 builtinID == X86::BI__builtin_ia32_movntss)
2429 src = builder.createExtractElement(loc, vec: ops[1], idx: 0);
2430
2431 cir::StoreOp so =
2432 builder.createStore(loc, val: src, dst: dest,
2433 /*isVolatile=*/false, /*isNontemporal=*/true);
2434 return so.getValue();
2435 }
2436 case X86::BI__builtin_ia32_vprotbi:
2437 case X86::BI__builtin_ia32_vprotwi:
2438 case X86::BI__builtin_ia32_vprotdi:
2439 case X86::BI__builtin_ia32_vprotqi:
2440 case X86::BI__builtin_ia32_prold128:
2441 case X86::BI__builtin_ia32_prold256:
2442 case X86::BI__builtin_ia32_prold512:
2443 case X86::BI__builtin_ia32_prolq128:
2444 case X86::BI__builtin_ia32_prolq256:
2445 case X86::BI__builtin_ia32_prolq512:
2446 return emitX86FunnelShift(builder, location: getLoc(srcLoc: expr->getExprLoc()), op0&: ops[0],
2447 op1&: ops[0], amt&: ops[1], isRight: false);
2448 case X86::BI__builtin_ia32_prord128:
2449 case X86::BI__builtin_ia32_prord256:
2450 case X86::BI__builtin_ia32_prord512:
2451 case X86::BI__builtin_ia32_prorq128:
2452 case X86::BI__builtin_ia32_prorq256:
2453 case X86::BI__builtin_ia32_prorq512:
2454 return emitX86FunnelShift(builder, location: getLoc(srcLoc: expr->getExprLoc()), op0&: ops[0],
2455 op1&: ops[0], amt&: ops[1], isRight: true);
2456 case X86::BI__builtin_ia32_selectb_128:
2457 case X86::BI__builtin_ia32_selectb_256:
2458 case X86::BI__builtin_ia32_selectb_512:
2459 case X86::BI__builtin_ia32_selectw_128:
2460 case X86::BI__builtin_ia32_selectw_256:
2461 case X86::BI__builtin_ia32_selectw_512:
2462 case X86::BI__builtin_ia32_selectd_128:
2463 case X86::BI__builtin_ia32_selectd_256:
2464 case X86::BI__builtin_ia32_selectd_512:
2465 case X86::BI__builtin_ia32_selectq_128:
2466 case X86::BI__builtin_ia32_selectq_256:
2467 case X86::BI__builtin_ia32_selectq_512:
2468 case X86::BI__builtin_ia32_selectph_128:
2469 case X86::BI__builtin_ia32_selectph_256:
2470 case X86::BI__builtin_ia32_selectph_512:
2471 case X86::BI__builtin_ia32_selectpbf_128:
2472 case X86::BI__builtin_ia32_selectpbf_256:
2473 case X86::BI__builtin_ia32_selectpbf_512:
2474 case X86::BI__builtin_ia32_selectps_128:
2475 case X86::BI__builtin_ia32_selectps_256:
2476 case X86::BI__builtin_ia32_selectps_512:
2477 case X86::BI__builtin_ia32_selectpd_128:
2478 case X86::BI__builtin_ia32_selectpd_256:
2479 case X86::BI__builtin_ia32_selectpd_512:
2480 return emitX86Select(builder, loc: getLoc(srcLoc: expr->getExprLoc()), mask: ops[0], op0: ops[1],
2481 op1: ops[2]);
2482 case X86::BI__builtin_ia32_selectsh_128:
2483 case X86::BI__builtin_ia32_selectsbf_128:
2484 case X86::BI__builtin_ia32_selectss_128:
2485 case X86::BI__builtin_ia32_selectsd_128: {
2486 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2487 mlir::Value scalar1 =
2488 builder.createExtractElement(loc, vec: ops[1], idx: uint64_t(0));
2489 mlir::Value scalar2 =
2490 builder.createExtractElement(loc, vec: ops[2], idx: uint64_t(0));
2491 mlir::Value result =
2492 emitX86ScalarSelect(builder, loc, mask: ops[0], op0: scalar1, op1: scalar2);
2493 return builder.createInsertElement(loc, vec: ops[1], newElt: result, idx: uint64_t(0));
2494 }
2495 case X86::BI__builtin_ia32_cmpb128_mask:
2496 case X86::BI__builtin_ia32_cmpb256_mask:
2497 case X86::BI__builtin_ia32_cmpb512_mask:
2498 case X86::BI__builtin_ia32_cmpw128_mask:
2499 case X86::BI__builtin_ia32_cmpw256_mask:
2500 case X86::BI__builtin_ia32_cmpw512_mask:
2501 case X86::BI__builtin_ia32_cmpd128_mask:
2502 case X86::BI__builtin_ia32_cmpd256_mask:
2503 case X86::BI__builtin_ia32_cmpd512_mask:
2504 case X86::BI__builtin_ia32_cmpq128_mask:
2505 case X86::BI__builtin_ia32_cmpq256_mask:
2506 case X86::BI__builtin_ia32_cmpq512_mask:
2507 case X86::BI__builtin_ia32_ucmpb128_mask:
2508 case X86::BI__builtin_ia32_ucmpb256_mask:
2509 case X86::BI__builtin_ia32_ucmpb512_mask:
2510 case X86::BI__builtin_ia32_ucmpw128_mask:
2511 case X86::BI__builtin_ia32_ucmpw256_mask:
2512 case X86::BI__builtin_ia32_ucmpw512_mask:
2513 case X86::BI__builtin_ia32_ucmpd128_mask:
2514 case X86::BI__builtin_ia32_ucmpd256_mask:
2515 case X86::BI__builtin_ia32_ucmpd512_mask:
2516 case X86::BI__builtin_ia32_ucmpq128_mask:
2517 case X86::BI__builtin_ia32_ucmpq256_mask:
2518 case X86::BI__builtin_ia32_ucmpq512_mask: {
2519 int64_t cc = CIRGenFunction::getZExtIntValueFromConstOp(val: ops[2]) & 0x7;
2520 return emitX86MaskedCompare(builder, cc, isSigned: 1, ops,
2521 loc: getLoc(srcLoc: expr->getExprLoc()));
2522 }
2523 case X86::BI__builtin_ia32_vpcomb:
2524 case X86::BI__builtin_ia32_vpcomw:
2525 case X86::BI__builtin_ia32_vpcomd:
2526 case X86::BI__builtin_ia32_vpcomq:
2527 return emitX86vpcom(builder, loc: getLoc(srcLoc: expr->getExprLoc()), ops, isSigned: true);
2528 case X86::BI__builtin_ia32_vpcomub:
2529 case X86::BI__builtin_ia32_vpcomuw:
2530 case X86::BI__builtin_ia32_vpcomud:
2531 case X86::BI__builtin_ia32_vpcomuq:
2532 return emitX86vpcom(builder, loc: getLoc(srcLoc: expr->getExprLoc()), ops, isSigned: false);
2533 case X86::BI__builtin_ia32_kortestcqi:
2534 case X86::BI__builtin_ia32_kortestchi:
2535 case X86::BI__builtin_ia32_kortestcsi:
2536 case X86::BI__builtin_ia32_kortestcdi: {
2537 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2538 cir::IntType ty = cast<cir::IntType>(Val: ops[0].getType());
2539 mlir::Value allOnesOp =
2540 builder.getConstAPInt(loc, typ: ty, val: APInt::getAllOnes(numBits: ty.getWidth()));
2541 mlir::Value orOp = emitX86MaskLogic<cir::OrOp>(builder, loc, ops);
2542 mlir::Value cmp =
2543 cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::eq, lhs: orOp, rhs: allOnesOp);
2544 return builder.createCast(kind: cir::CastKind::bool_to_int, src: cmp,
2545 newTy: cgm.convertType(type: expr->getType()));
2546 }
2547 case X86::BI__builtin_ia32_kortestzqi:
2548 case X86::BI__builtin_ia32_kortestzhi:
2549 case X86::BI__builtin_ia32_kortestzsi:
2550 case X86::BI__builtin_ia32_kortestzdi: {
2551 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2552 cir::IntType ty = cast<cir::IntType>(Val: ops[0].getType());
2553 mlir::Value allZerosOp = builder.getNullValue(ty, loc).getResult();
2554 mlir::Value orOp = emitX86MaskLogic<cir::OrOp>(builder, loc, ops);
2555 mlir::Value cmp =
2556 cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::eq, lhs: orOp, rhs: allZerosOp);
2557 return builder.createCast(kind: cir::CastKind::bool_to_int, src: cmp,
2558 newTy: cgm.convertType(type: expr->getType()));
2559 }
2560 case X86::BI__builtin_ia32_ktestcqi:
2561 return emitX86MaskTest(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2562 intrinsicName: "x86.avx512.ktestc.b", ops);
2563 case X86::BI__builtin_ia32_ktestzqi:
2564 return emitX86MaskTest(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2565 intrinsicName: "x86.avx512.ktestz.b", ops);
2566 case X86::BI__builtin_ia32_ktestchi:
2567 return emitX86MaskTest(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2568 intrinsicName: "x86.avx512.ktestc.w", ops);
2569 case X86::BI__builtin_ia32_ktestzhi:
2570 return emitX86MaskTest(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2571 intrinsicName: "x86.avx512.ktestz.w", ops);
2572 case X86::BI__builtin_ia32_ktestcsi:
2573 return emitX86MaskTest(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2574 intrinsicName: "x86.avx512.ktestc.d", ops);
2575 case X86::BI__builtin_ia32_ktestzsi:
2576 return emitX86MaskTest(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2577 intrinsicName: "x86.avx512.ktestz.d", ops);
2578 case X86::BI__builtin_ia32_ktestcdi:
2579 return emitX86MaskTest(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2580 intrinsicName: "x86.avx512.ktestc.q", ops);
2581 case X86::BI__builtin_ia32_ktestzdi:
2582 return emitX86MaskTest(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2583 intrinsicName: "x86.avx512.ktestz.q", ops);
2584 case X86::BI__builtin_ia32_kaddqi:
2585 return emitX86MaskAddLogic(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2586 intrinsicName: "x86.avx512.kadd.b", ops);
2587 case X86::BI__builtin_ia32_kaddhi:
2588 return emitX86MaskAddLogic(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2589 intrinsicName: "x86.avx512.kadd.w", ops);
2590 case X86::BI__builtin_ia32_kaddsi:
2591 return emitX86MaskAddLogic(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2592 intrinsicName: "x86.avx512.kadd.d", ops);
2593 case X86::BI__builtin_ia32_kadddi:
2594 return emitX86MaskAddLogic(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2595 intrinsicName: "x86.avx512.kadd.q", ops);
2596 case X86::BI__builtin_ia32_kandqi:
2597 case X86::BI__builtin_ia32_kandhi:
2598 case X86::BI__builtin_ia32_kandsi:
2599 case X86::BI__builtin_ia32_kanddi:
2600 return emitX86MaskLogic<cir::AndOp>(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2601 ops);
2602 case X86::BI__builtin_ia32_kandnqi:
2603 case X86::BI__builtin_ia32_kandnhi:
2604 case X86::BI__builtin_ia32_kandnsi:
2605 case X86::BI__builtin_ia32_kandndi:
2606 return emitX86MaskLogic<cir::AndOp>(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2607 ops, /*invertLHS=*/true);
2608 case X86::BI__builtin_ia32_korqi:
2609 case X86::BI__builtin_ia32_korhi:
2610 case X86::BI__builtin_ia32_korsi:
2611 case X86::BI__builtin_ia32_kordi:
2612 return emitX86MaskLogic<cir::OrOp>(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2613 ops);
2614 case X86::BI__builtin_ia32_kxnorqi:
2615 case X86::BI__builtin_ia32_kxnorhi:
2616 case X86::BI__builtin_ia32_kxnorsi:
2617 case X86::BI__builtin_ia32_kxnordi:
2618 return emitX86MaskLogic<cir::XorOp>(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2619 ops, /*invertLHS=*/true);
2620 case X86::BI__builtin_ia32_kxorqi:
2621 case X86::BI__builtin_ia32_kxorhi:
2622 case X86::BI__builtin_ia32_kxorsi:
2623 case X86::BI__builtin_ia32_kxordi:
2624 return emitX86MaskLogic<cir::XorOp>(builder, loc: getLoc(srcLoc: expr->getExprLoc()),
2625 ops);
2626 case X86::BI__builtin_ia32_knotqi:
2627 case X86::BI__builtin_ia32_knothi:
2628 case X86::BI__builtin_ia32_knotsi:
2629 case X86::BI__builtin_ia32_knotdi: {
2630 cir::IntType intTy = cast<cir::IntType>(Val: ops[0].getType());
2631 unsigned numElts = intTy.getWidth();
2632 mlir::Value resVec =
2633 getMaskVecValue(builder, loc: getLoc(srcLoc: expr->getExprLoc()), mask: ops[0], numElems: numElts);
2634 return builder.createBitcast(src: builder.createNot(value: resVec), newTy: ops[0].getType());
2635 }
2636 case X86::BI__builtin_ia32_kmovb:
2637 case X86::BI__builtin_ia32_kmovw:
2638 case X86::BI__builtin_ia32_kmovd:
2639 case X86::BI__builtin_ia32_kmovq: {
2640 // Bitcast to vXi1 type and then back to integer. This gets the mask
2641 // register type into the IR, but might be optimized out depending on
2642 // what's around it.
2643 cir::IntType intTy = cast<cir::IntType>(Val: ops[0].getType());
2644 unsigned numElts = intTy.getWidth();
2645 mlir::Value resVec =
2646 getMaskVecValue(builder, loc: getLoc(srcLoc: expr->getExprLoc()), mask: ops[0], numElems: numElts);
2647 return builder.createBitcast(src: resVec, newTy: ops[0].getType());
2648 }
2649 case X86::BI__builtin_ia32_sqrtsh_round_mask:
2650 case X86::BI__builtin_ia32_sqrtsd_round_mask:
2651 case X86::BI__builtin_ia32_sqrtss_round_mask:
2652 cgm.errorNYI(expr->getSourceRange(),
2653 std::string("unimplemented X86 builtin call: ") +
2654 getContext().BuiltinInfo.getName(ID: builtinID));
2655 return mlir::Value{};
2656 case X86::BI__builtin_ia32_sqrtph512:
2657 case X86::BI__builtin_ia32_sqrtps512:
2658 case X86::BI__builtin_ia32_sqrtpd512: {
2659 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2660 mlir::Value arg = ops[0];
2661 return cir::SqrtOp::create(builder, location: loc, src: arg).getResult();
2662 }
2663 case X86::BI__builtin_ia32_pmulhw128:
2664 case X86::BI__builtin_ia32_pmulhw256:
2665 case X86::BI__builtin_ia32_pmulhw512:
2666 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()), str: "smulh",
2667 resTy: ops[0].getType(), op&: ops);
2668 case X86::BI__builtin_ia32_pmulhuw128:
2669 case X86::BI__builtin_ia32_pmulhuw256:
2670 case X86::BI__builtin_ia32_pmulhuw512:
2671 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()), str: "umulh",
2672 resTy: ops[0].getType(), op&: ops);
2673 case X86::BI__builtin_ia32_pmuludq128:
2674 case X86::BI__builtin_ia32_pmuludq256:
2675 case X86::BI__builtin_ia32_pmuludq512: {
2676 unsigned opTypePrimitiveSizeInBits =
2677 cgm.getDataLayout().getTypeSizeInBits(ty: ops[0].getType());
2678 return emitX86Muldq(builder, loc: getLoc(srcLoc: expr->getExprLoc()), /*isSigned*/ false,
2679 ops, opTypePrimitiveSizeInBits);
2680 }
2681 case X86::BI__builtin_ia32_pmuldq128:
2682 case X86::BI__builtin_ia32_pmuldq256:
2683 case X86::BI__builtin_ia32_pmuldq512: {
2684 unsigned opTypePrimitiveSizeInBits =
2685 cgm.getDataLayout().getTypeSizeInBits(ty: ops[0].getType());
2686 return emitX86Muldq(builder, loc: getLoc(srcLoc: expr->getExprLoc()), /*isSigned*/ true,
2687 ops, opTypePrimitiveSizeInBits);
2688 }
2689 case X86::BI__builtin_ia32_pternlogd512_mask:
2690 case X86::BI__builtin_ia32_pternlogq512_mask:
2691 case X86::BI__builtin_ia32_pternlogd128_mask:
2692 case X86::BI__builtin_ia32_pternlogd256_mask:
2693 case X86::BI__builtin_ia32_pternlogq128_mask:
2694 case X86::BI__builtin_ia32_pternlogq256_mask:
2695 case X86::BI__builtin_ia32_pternlogd512_maskz:
2696 case X86::BI__builtin_ia32_pternlogq512_maskz:
2697 case X86::BI__builtin_ia32_pternlogd128_maskz:
2698 case X86::BI__builtin_ia32_pternlogd256_maskz:
2699 case X86::BI__builtin_ia32_pternlogq128_maskz:
2700 case X86::BI__builtin_ia32_pternlogq256_maskz:
2701 cgm.errorNYI(expr->getSourceRange(),
2702 std::string("unimplemented X86 builtin call: ") +
2703 getContext().BuiltinInfo.getName(ID: builtinID));
2704 return mlir::Value{};
2705 case X86::BI__builtin_ia32_vpshldd128:
2706 case X86::BI__builtin_ia32_vpshldd256:
2707 case X86::BI__builtin_ia32_vpshldd512:
2708 case X86::BI__builtin_ia32_vpshldq128:
2709 case X86::BI__builtin_ia32_vpshldq256:
2710 case X86::BI__builtin_ia32_vpshldq512:
2711 case X86::BI__builtin_ia32_vpshldw128:
2712 case X86::BI__builtin_ia32_vpshldw256:
2713 case X86::BI__builtin_ia32_vpshldw512:
2714 return emitX86FunnelShift(builder, location: getLoc(srcLoc: expr->getExprLoc()), op0&: ops[0],
2715 op1&: ops[1], amt&: ops[2], isRight: false);
2716 case X86::BI__builtin_ia32_vpshrdd128:
2717 case X86::BI__builtin_ia32_vpshrdd256:
2718 case X86::BI__builtin_ia32_vpshrdd512:
2719 case X86::BI__builtin_ia32_vpshrdq128:
2720 case X86::BI__builtin_ia32_vpshrdq256:
2721 case X86::BI__builtin_ia32_vpshrdq512:
2722 case X86::BI__builtin_ia32_vpshrdw128:
2723 case X86::BI__builtin_ia32_vpshrdw256:
2724 case X86::BI__builtin_ia32_vpshrdw512:
2725 // Ops 0 and 1 are swapped.
2726 return emitX86FunnelShift(builder, location: getLoc(srcLoc: expr->getExprLoc()), op0&: ops[1],
2727 op1&: ops[0], amt&: ops[2], isRight: true);
2728 case X86::BI__builtin_ia32_reduce_fadd_pd512:
2729 case X86::BI__builtin_ia32_reduce_fadd_ps512:
2730 case X86::BI__builtin_ia32_reduce_fadd_ph512:
2731 case X86::BI__builtin_ia32_reduce_fadd_ph256:
2732 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
2733 CIRGenFPOptionsRAII FPOptsRAII(*this, expr);
2734 cir::FastMathFlags fastMathFlags = getCurrentFastMathFlags();
2735 fastMathFlags |= cir::FastMathFlags::reassoc;
2736 cir::FastMathFlagsAttr fastMath = getFastMathFlagsAttr(fastMathFlags);
2737 return cir::VecReduceFAddOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()),
2738 accumulator: ops[0], input: ops[1], fastmath_flags: fastMath)
2739 .getResult();
2740 }
2741 case X86::BI__builtin_ia32_reduce_fmul_pd512:
2742 case X86::BI__builtin_ia32_reduce_fmul_ps512:
2743 case X86::BI__builtin_ia32_reduce_fmul_ph512:
2744 case X86::BI__builtin_ia32_reduce_fmul_ph256:
2745 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
2746 CIRGenFPOptionsRAII FPOptsRAII(*this, expr);
2747 cir::FastMathFlags fastMathFlags = getCurrentFastMathFlags();
2748 fastMathFlags |= cir::FastMathFlags::reassoc;
2749 cir::FastMathFlagsAttr fastMath = getFastMathFlagsAttr(fastMathFlags);
2750 return cir::VecReduceFMulOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()),
2751 accumulator: ops[0], input: ops[1], fastmath_flags: fastMath)
2752 .getResult();
2753 }
2754 case X86::BI__builtin_ia32_reduce_fmax_pd512:
2755 case X86::BI__builtin_ia32_reduce_fmax_ps512:
2756 case X86::BI__builtin_ia32_reduce_fmax_ph512:
2757 case X86::BI__builtin_ia32_reduce_fmax_ph256:
2758 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
2759 CIRGenFPOptionsRAII FPOptsRAII(*this, expr);
2760 cir::FastMathFlags fastMathFlags = getCurrentFastMathFlags();
2761 fastMathFlags |= cir::FastMathFlags::nnan;
2762 cir::FastMathFlagsAttr fastMath = getFastMathFlagsAttr(fastMathFlags);
2763 return cir::VecReduceFMaxOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()),
2764 input: ops[0], fastmath_flags: fastMath)
2765 .getResult();
2766 }
2767 case X86::BI__builtin_ia32_reduce_fmin_pd512:
2768 case X86::BI__builtin_ia32_reduce_fmin_ps512:
2769 case X86::BI__builtin_ia32_reduce_fmin_ph512:
2770 case X86::BI__builtin_ia32_reduce_fmin_ph256:
2771 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
2772 CIRGenFPOptionsRAII FPOptsRAII(*this, expr);
2773 cir::FastMathFlags fastMathFlags = getCurrentFastMathFlags();
2774 fastMathFlags |= cir::FastMathFlags::nnan;
2775 cir::FastMathFlagsAttr fastMath = getFastMathFlagsAttr(fastMathFlags);
2776 return cir::VecReduceFMinOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()),
2777 input: ops[0], fastmath_flags: fastMath)
2778 .getResult();
2779 }
2780 case X86::BI__builtin_ia32_rdrand16_step:
2781 case X86::BI__builtin_ia32_rdrand32_step:
2782 case X86::BI__builtin_ia32_rdrand64_step:
2783 case X86::BI__builtin_ia32_rdseed16_step:
2784 case X86::BI__builtin_ia32_rdseed32_step:
2785 case X86::BI__builtin_ia32_rdseed64_step: {
2786 llvm::StringRef intrinsicName;
2787 switch (builtinID) {
2788 default:
2789 llvm_unreachable("Unsupported intrinsic!");
2790 case X86::BI__builtin_ia32_rdrand16_step:
2791 intrinsicName = "x86.rdrand.16";
2792 break;
2793 case X86::BI__builtin_ia32_rdrand32_step:
2794 intrinsicName = "x86.rdrand.32";
2795 break;
2796 case X86::BI__builtin_ia32_rdrand64_step:
2797 intrinsicName = "x86.rdrand.64";
2798 break;
2799 case X86::BI__builtin_ia32_rdseed16_step:
2800 intrinsicName = "x86.rdseed.16";
2801 break;
2802 case X86::BI__builtin_ia32_rdseed32_step:
2803 intrinsicName = "x86.rdseed.32";
2804 break;
2805 case X86::BI__builtin_ia32_rdseed64_step:
2806 intrinsicName = "x86.rdseed.64";
2807 break;
2808 }
2809
2810 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2811 mlir::Type randTy = cast<cir::PointerType>(Val: ops[0].getType()).getPointee();
2812 llvm::SmallVector<mlir::Type, 2> resultTypes = {randTy,
2813 builder.getUInt32Ty()};
2814 cir::StructType resRecord = cir::StructType::get(
2815 context: &getMLIRContext(), members: resultTypes, /*packed=*/false,
2816 /*is_class=*/false, member_kinds: cir::RecordType::getAllDataKinds(members: resultTypes));
2817
2818 mlir::Value call =
2819 builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: resRecord);
2820 mlir::Value rand =
2821 cir::ExtractMemberOp::create(builder, location: loc, result: randTy, record: call, index: 0);
2822 builder.CIRBaseBuilderTy::createStore(loc, val: rand, dst: ops[0]);
2823
2824 return cir::ExtractMemberOp::create(builder, location: loc, result: builder.getUInt32Ty(),
2825 record: call, index: 1);
2826 }
2827 case X86::BI__builtin_ia32_addcarryx_u32:
2828 case X86::BI__builtin_ia32_addcarryx_u64:
2829 case X86::BI__builtin_ia32_subborrow_u32:
2830 case X86::BI__builtin_ia32_subborrow_u64: {
2831 llvm::StringRef intrinsicName;
2832 switch (builtinID) {
2833 default:
2834 llvm_unreachable("Unsupported intrinsic!");
2835 case X86::BI__builtin_ia32_addcarryx_u32:
2836 intrinsicName = "x86.addcarry.32";
2837 break;
2838 case X86::BI__builtin_ia32_addcarryx_u64:
2839 intrinsicName = "x86.addcarry.64";
2840 break;
2841 case X86::BI__builtin_ia32_subborrow_u32:
2842 intrinsicName = "x86.subborrow.32";
2843 break;
2844 case X86::BI__builtin_ia32_subborrow_u64:
2845 intrinsicName = "x86.subborrow.64";
2846 break;
2847 }
2848
2849 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2850 mlir::Type carryTy = ops[0].getType();
2851 mlir::Type valueTy = ops[1].getType();
2852 llvm::SmallVector<mlir::Type, 2> resultTypes = {carryTy, valueTy};
2853 cir::StructType resRecord =
2854 builder.getAnonRecordTy(members: resultTypes, /*packed=*/false,
2855 memberKinds: cir::RecordType::getAllDataKinds(members: resultTypes));
2856
2857 mlir::Value call =
2858 builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: resRecord,
2859 op: mlir::ValueRange{ops[0], ops[1], ops[2]});
2860 mlir::Value result =
2861 cir::ExtractMemberOp::create(builder, location: loc, result: valueTy, record: call, index: 1);
2862 builder.CIRBaseBuilderTy::createStore(loc, val: result, dst: ops[3]);
2863
2864 return cir::ExtractMemberOp::create(builder, location: loc, result: carryTy, record: call, index: 0);
2865 }
2866 case X86::BI__builtin_ia32_fpclassps128_mask:
2867 case X86::BI__builtin_ia32_fpclassps256_mask:
2868 case X86::BI__builtin_ia32_fpclassps512_mask:
2869 case X86::BI__builtin_ia32_vfpclassbf16128_mask:
2870 case X86::BI__builtin_ia32_vfpclassbf16256_mask:
2871 case X86::BI__builtin_ia32_vfpclassbf16512_mask:
2872 case X86::BI__builtin_ia32_fpclassph128_mask:
2873 case X86::BI__builtin_ia32_fpclassph256_mask:
2874 case X86::BI__builtin_ia32_fpclassph512_mask:
2875 case X86::BI__builtin_ia32_fpclasspd128_mask:
2876 case X86::BI__builtin_ia32_fpclasspd256_mask:
2877 case X86::BI__builtin_ia32_fpclasspd512_mask:
2878 return emitX86Fpclass(builder, loc: getLoc(srcLoc: expr->getExprLoc()), builtinID, ops);
2879 case X86::BI__builtin_ia32_vp2intersect_q_512:
2880 case X86::BI__builtin_ia32_vp2intersect_q_256:
2881 case X86::BI__builtin_ia32_vp2intersect_q_128:
2882 case X86::BI__builtin_ia32_vp2intersect_d_512:
2883 case X86::BI__builtin_ia32_vp2intersect_d_256:
2884 case X86::BI__builtin_ia32_vp2intersect_d_128: {
2885 unsigned numElts = cast<cir::VectorType>(Val: ops[0].getType()).getFixedSize();
2886 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2887 StringRef intrinsicName;
2888
2889 switch (builtinID) {
2890 default:
2891 llvm_unreachable("Unexpected builtin");
2892 case X86::BI__builtin_ia32_vp2intersect_q_512:
2893 intrinsicName = "x86.avx512.vp2intersect.q.512";
2894 break;
2895 case X86::BI__builtin_ia32_vp2intersect_q_256:
2896 intrinsicName = "x86.avx512.vp2intersect.q.256";
2897 break;
2898 case X86::BI__builtin_ia32_vp2intersect_q_128:
2899 intrinsicName = "x86.avx512.vp2intersect.q.128";
2900 break;
2901 case X86::BI__builtin_ia32_vp2intersect_d_512:
2902 intrinsicName = "x86.avx512.vp2intersect.d.512";
2903 break;
2904 case X86::BI__builtin_ia32_vp2intersect_d_256:
2905 intrinsicName = "x86.avx512.vp2intersect.d.256";
2906 break;
2907 case X86::BI__builtin_ia32_vp2intersect_d_128:
2908 intrinsicName = "x86.avx512.vp2intersect.d.128";
2909 break;
2910 }
2911
2912 auto resVector = cir::VectorType::getFixed(elementType: builder.getSIntNTy(n: 1), size: numElts);
2913
2914 mlir::Type resMembers[] = {resVector, resVector};
2915 cir::StructType resRecord = cir::StructType::get(
2916 context: &getMLIRContext(), members: resMembers, /*packed=*/false, /*is_class=*/false,
2917 member_kinds: cir::RecordType::getAllDataKinds(members: resMembers));
2918
2919 mlir::Value call = builder.emitIntrinsicCallOp(
2920 loc: getLoc(srcLoc: expr->getExprLoc()), str: intrinsicName, resTy: resRecord,
2921 op: mlir::ValueRange{ops[0], ops[1]});
2922 mlir::Value result =
2923 cir::ExtractMemberOp::create(builder, location: loc, result: resVector, record: call, index: 0);
2924 result = emitX86MaskedCompareResult(builder, cmp: result, numElts, maskIn: nullptr, loc);
2925 Address addr = Address(
2926 ops[2], clang::CharUnits::fromQuantity(Quantity: std::max(a: 1U, b: numElts / 8)));
2927 builder.createStore(loc, val: result, dst: addr);
2928
2929 result = cir::ExtractMemberOp::create(builder, location: loc, result: resVector, record: call, index: 1);
2930 result = emitX86MaskedCompareResult(builder, cmp: result, numElts, maskIn: nullptr, loc);
2931 addr = Address(ops[3],
2932 clang::CharUnits::fromQuantity(Quantity: std::max(a: 1U, b: numElts / 8)));
2933 builder.createStore(loc, val: result, dst: addr);
2934 return mlir::Value{};
2935 }
2936 case X86::BI__builtin_ia32_vpmultishiftqb128:
2937 case X86::BI__builtin_ia32_vpmultishiftqb256:
2938 case X86::BI__builtin_ia32_vpmultishiftqb512:
2939 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
2940 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
2941 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
2942 cgm.errorNYI(expr->getSourceRange(),
2943 std::string("unimplemented X86 builtin call: ") +
2944 getContext().BuiltinInfo.getName(ID: builtinID));
2945 return {};
2946 case X86::BI__builtin_ia32_cmpeqps:
2947 case X86::BI__builtin_ia32_cmpeqpd:
2948 return emitVectorFCmp(cgf&: *this, expr: *expr, ops, pred: cir::FCmpOpKind::oeq,
2949 /*signaling=*/false);
2950 case X86::BI__builtin_ia32_cmpltps:
2951 case X86::BI__builtin_ia32_cmpltpd:
2952 return emitVectorFCmp(cgf&: *this, expr: *expr, ops, pred: cir::FCmpOpKind::olt,
2953 /*signaling=*/true);
2954 case X86::BI__builtin_ia32_cmpleps:
2955 case X86::BI__builtin_ia32_cmplepd:
2956 return emitVectorFCmp(cgf&: *this, expr: *expr, ops, pred: cir::FCmpOpKind::ole,
2957 /*signaling=*/true);
2958 case X86::BI__builtin_ia32_cmpunordps:
2959 case X86::BI__builtin_ia32_cmpunordpd:
2960 return emitVectorFCmp(cgf&: *this, expr: *expr, ops, pred: cir::FCmpOpKind::uno,
2961 /*signaling=*/false);
2962 case X86::BI__builtin_ia32_cmpneqps:
2963 case X86::BI__builtin_ia32_cmpneqpd:
2964 return emitVectorFCmp(cgf&: *this, expr: *expr, ops, pred: cir::FCmpOpKind::une,
2965 /*signaling=*/false);
2966 case X86::BI__builtin_ia32_cmpnltps:
2967 case X86::BI__builtin_ia32_cmpnltpd:
2968 return emitVectorFCmp(cgf&: *this, expr: *expr, ops, pred: cir::FCmpOpKind::uge,
2969 /*signaling=*/true);
2970 case X86::BI__builtin_ia32_cmpnleps:
2971 case X86::BI__builtin_ia32_cmpnlepd:
2972 return emitVectorFCmp(cgf&: *this, expr: *expr, ops, pred: cir::FCmpOpKind::ugt,
2973 /*signaling=*/true);
2974 case X86::BI__builtin_ia32_cmpordps:
2975 case X86::BI__builtin_ia32_cmpordpd:
2976 return emitVectorFCmp(cgf&: *this, expr: *expr, ops, pred: cir::FCmpOpKind::ord,
2977 /*signaling=*/false);
2978 case X86::BI__builtin_ia32_cmpph128_mask:
2979 case X86::BI__builtin_ia32_cmpph256_mask:
2980 case X86::BI__builtin_ia32_cmpph512_mask:
2981 case X86::BI__builtin_ia32_cmpps128_mask:
2982 case X86::BI__builtin_ia32_cmpps256_mask:
2983 case X86::BI__builtin_ia32_cmpps512_mask:
2984 case X86::BI__builtin_ia32_cmppd128_mask:
2985 case X86::BI__builtin_ia32_cmppd256_mask:
2986 case X86::BI__builtin_ia32_cmppd512_mask:
2987 case X86::BI__builtin_ia32_vcmpbf16512_mask:
2988 case X86::BI__builtin_ia32_vcmpbf16256_mask:
2989 case X86::BI__builtin_ia32_vcmpbf16128_mask:
2990 isMaskFCmp = true;
2991 [[fallthrough]];
2992 case X86::BI__builtin_ia32_cmpps:
2993 case X86::BI__builtin_ia32_cmpps256:
2994 case X86::BI__builtin_ia32_cmppd:
2995 case X86::BI__builtin_ia32_cmppd256: {
2996 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2997
2998 // The third argument is the comparison condition.
2999 unsigned cc =
3000 ops[2].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue() &
3001 0x1f;
3002
3003 X86FCmpInfo cmpInfo = getX86FCmpInfo(cc);
3004
3005 auto opsTy = cast<cir::VectorType>(Val: ops[0].getType());
3006 unsigned numElts = opsTy.getFixedSize();
3007
3008 // If the predicate is true or false, or the builtin is mask enabled, and
3009 // we're using constrained intrinsics, there is no compare operation we can
3010 // use. Just use the legacy X86 specific intrinsic.
3011 if (builder.getIsFPConstrained() &&
3012 (cmpInfo.pred == cir::FCmpOpKind::_true ||
3013 cmpInfo.pred == cir::FCmpOpKind::_false || isMaskFCmp)) {
3014 llvm::StringRef intrinsicName = getX86FCmpIntrinsicName(builtinID);
3015
3016 if (isMaskFCmp) {
3017 ops[3] = getMaskVecValue(builder, loc, mask: ops[3], numElems: numElts);
3018 mlir::Value cmp = builder.emitIntrinsicCallOp(
3019 loc, str: intrinsicName,
3020 resTy: cir::VectorType::getFixed(elementType: builder.getSIntNTy(n: 1), size: numElts), op&: ops);
3021 return emitX86MaskedCompareResult(builder, cmp, numElts, maskIn: nullptr, loc);
3022 }
3023
3024 return builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: ops[0].getType(),
3025 op&: ops);
3026 }
3027
3028 unsigned width = builder.getCIRIntOrFloatBitWidth(eltTy: opsTy.getElementType());
3029 cir::IntType cmpEltTy = builder.getSIntNTy(n: isMaskFCmp ? 1 : width);
3030 auto cmpTy = cir::VectorType::getFixed(elementType: cmpEltTy, size: numElts);
3031
3032 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, expr);
3033 mlir::Value cmp = builder.createVecFCompare(
3034 loc, resultTy: cmpTy, kind: cmpInfo.pred, lhs: ops[0], rhs: ops[1], signaling: cmpInfo.isSignaling);
3035
3036 if (isMaskFCmp)
3037 return emitX86MaskedCompareResult(builder, cmp, numElts, maskIn: ops[3], loc);
3038
3039 return builder.createBitcast(src: cmp, newTy: opsTy);
3040 }
3041 case X86::BI__builtin_ia32_cmpeqss:
3042 case X86::BI__builtin_ia32_cmpltss:
3043 case X86::BI__builtin_ia32_cmpless:
3044 case X86::BI__builtin_ia32_cmpunordss:
3045 case X86::BI__builtin_ia32_cmpneqss:
3046 case X86::BI__builtin_ia32_cmpnltss:
3047 case X86::BI__builtin_ia32_cmpnless:
3048 case X86::BI__builtin_ia32_cmpordss:
3049 case X86::BI__builtin_ia32_cmpeqsd:
3050 case X86::BI__builtin_ia32_cmpltsd:
3051 case X86::BI__builtin_ia32_cmplesd:
3052 case X86::BI__builtin_ia32_cmpunordsd:
3053 case X86::BI__builtin_ia32_cmpneqsd:
3054 case X86::BI__builtin_ia32_cmpnltsd:
3055 case X86::BI__builtin_ia32_cmpnlesd:
3056 case X86::BI__builtin_ia32_cmpordsd:
3057 cgm.errorNYI(expr->getSourceRange(),
3058 std::string("unimplemented X86 builtin call: ") +
3059 getContext().BuiltinInfo.getName(ID: builtinID));
3060 return {};
3061 case X86::BI__builtin_ia32_vcvtph2ps_mask:
3062 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
3063 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
3064 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
3065 return emitX86CvtF16ToFloatExpr(builder, loc, ops,
3066 dstTy: convertType(t: expr->getType()));
3067 }
3068 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
3069 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
3070 cir::VectorType resTy = cast<cir::VectorType>(Val: convertType(t: expr->getType()));
3071
3072 cir::VectorType inputTy = cast<cir::VectorType>(Val: ops[0].getType());
3073 unsigned numElts = inputTy.getFixedSize();
3074
3075 mlir::Value mask = getMaskVecValue(builder, loc, mask: ops[2], numElems: numElts);
3076
3077 SmallVector<mlir::Value, 3> args;
3078 args.push_back(Elt: ops[0]);
3079 args.push_back(Elt: ops[1]);
3080 args.push_back(Elt: mask);
3081
3082 return builder.emitIntrinsicCallOp(
3083 loc, str: "x86.avx512bf16.mask.cvtneps2bf16.128", resTy, op&: args);
3084 }
3085 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
3086 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
3087 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
3088 cir::VectorType resTy = cast<cir::VectorType>(Val: convertType(t: expr->getType()));
3089 StringRef intrinsicName;
3090 if (builtinID == X86::BI__builtin_ia32_cvtneps2bf16_256_mask) {
3091 intrinsicName = "x86.avx512bf16.cvtneps2bf16.256";
3092 } else {
3093 assert(builtinID == X86::BI__builtin_ia32_cvtneps2bf16_512_mask);
3094 intrinsicName = "x86.avx512bf16.cvtneps2bf16.512";
3095 }
3096
3097 mlir::Value res = builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy,
3098 op: mlir::ValueRange{ops[0]});
3099
3100 return emitX86Select(builder, loc, mask: ops[2], op0: res, op1: ops[1]);
3101 }
3102 case X86::BI__cpuid:
3103 case X86::BI__cpuidex: {
3104 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
3105 mlir::Value subFuncId = builtinID == X86::BI__cpuidex
3106 ? ops[2]
3107 : builder.getConstInt(loc, t: sInt32Ty, c: 0);
3108 cir::CpuIdOp::create(builder, location: loc, /*cpuInfo=*/cpu_info: ops[0],
3109 /*functionId=*/function_id: ops[1], /*subFunctionId=*/sub_function_id: subFuncId);
3110 return mlir::Value{};
3111 }
3112 case X86::BI__emul:
3113 case X86::BI__emulu:
3114 case X86::BI__mulh:
3115 case X86::BI__umulh:
3116 case X86::BI_mul128:
3117 case X86::BI_umul128: {
3118 cgm.errorNYI(expr->getSourceRange(),
3119 std::string("unimplemented X86 builtin call: ") +
3120 getContext().BuiltinInfo.getName(ID: builtinID));
3121 return mlir::Value{};
3122 }
3123 case X86::BI__faststorefence: {
3124 cir::AtomicFenceOp::create(
3125 builder, location: getLoc(srcLoc: expr->getExprLoc()),
3126 ordering: cir::MemOrder::SequentiallyConsistent,
3127 syncscope: cir::SyncScopeKindAttr::get(context: &getMLIRContext(),
3128 value: cir::SyncScopeKind::System));
3129 return mlir::Value{};
3130 }
3131 case X86::BI__shiftleft128:
3132 case X86::BI__shiftright128: {
3133 // Flip low/high ops and zero-extend amount to matching type.
3134 // shiftleft128(Low, High, Amt) -> fshl(High, Low, Amt)
3135 // shiftright128(Low, High, Amt) -> fshr(High, Low, Amt)
3136 std::swap(a&: ops[0], b&: ops[1]);
3137
3138 // Zero-extend shift amount to i64 if needed
3139 auto amtTy = mlir::cast<cir::IntType>(Val: ops[2].getType());
3140 cir::IntType i64Ty = builder.getUInt64Ty();
3141
3142 if (amtTy != i64Ty)
3143 ops[2] = builder.createIntCast(src: ops[2], newTy: i64Ty);
3144
3145 const StringRef intrinsicName =
3146 (builtinID == X86::BI__shiftleft128) ? "fshl" : "fshr";
3147 return builder.emitIntrinsicCallOp(
3148 loc: getLoc(srcLoc: expr->getExprLoc()), str: intrinsicName, resTy: i64Ty,
3149 op: mlir::ValueRange{ops[0], ops[1], ops[2]});
3150 }
3151 case X86::BI_AddressOfReturnAddress: {
3152 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
3153 mlir::Value addr =
3154 cir::AddrOfReturnAddrOp::create(builder, location: loc, result: allocaInt8PtrTy);
3155 return builder.createCast(loc, kind: cir::CastKind::bitcast, src: addr, newTy: voidPtrTy);
3156 }
3157 case X86::BI__stosb:
3158 case X86::BI__ud2:
3159 case X86::BI__int2c:
3160 case X86::BI__readfsbyte:
3161 case X86::BI__readfsword:
3162 case X86::BI__readfsdword:
3163 case X86::BI__readfsqword:
3164 case X86::BI__readgsbyte:
3165 case X86::BI__readgsword:
3166 case X86::BI__readgsdword:
3167 case X86::BI__readgsqword: {
3168 cgm.errorNYI(expr->getSourceRange(),
3169 std::string("unimplemented X86 builtin call: ") +
3170 getContext().BuiltinInfo.getName(ID: builtinID));
3171 return mlir::Value{};
3172 }
3173 case X86::BI__builtin_ia32_encodekey128_u32: {
3174 return emitEncodeKey(context: &getMLIRContext(), builder, location: getLoc(srcLoc: expr->getExprLoc()),
3175 inputOperands: {ops[0], ops[1]}, outputOperand: ops[2], vecOutputCount: 6, intrinsicName: "x86.encodekey128", numResults: 3);
3176 }
3177 case X86::BI__builtin_ia32_encodekey256_u32: {
3178
3179 return emitEncodeKey(context: &getMLIRContext(), builder, location: getLoc(srcLoc: expr->getExprLoc()),
3180 inputOperands: {ops[0], ops[1], ops[2]}, outputOperand: ops[3], vecOutputCount: 7,
3181 intrinsicName: "x86.encodekey256", numResults: 4);
3182 }
3183
3184 case X86::BI__builtin_ia32_aesenc128kl_u8:
3185 case X86::BI__builtin_ia32_aesdec128kl_u8:
3186 case X86::BI__builtin_ia32_aesenc256kl_u8:
3187 case X86::BI__builtin_ia32_aesdec256kl_u8: {
3188 llvm::StringRef intrinsicName;
3189 switch (builtinID) {
3190 default:
3191 llvm_unreachable("Unexpected builtin");
3192 case X86::BI__builtin_ia32_aesenc128kl_u8:
3193 intrinsicName = "x86.aesenc128kl";
3194 break;
3195 case X86::BI__builtin_ia32_aesdec128kl_u8:
3196 intrinsicName = "x86.aesdec128kl";
3197 break;
3198 case X86::BI__builtin_ia32_aesenc256kl_u8:
3199 intrinsicName = "x86.aesenc256kl";
3200 break;
3201 case X86::BI__builtin_ia32_aesdec256kl_u8:
3202 intrinsicName = "x86.aesdec256kl";
3203 break;
3204 }
3205
3206 return emitX86Aes(builder, loc: getLoc(srcLoc: expr->getExprLoc()), intrinsicName,
3207 retType: convertType(t: expr->getType()), ops);
3208 }
3209 case X86::BI__builtin_ia32_aesencwide128kl_u8:
3210 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
3211 case X86::BI__builtin_ia32_aesencwide256kl_u8:
3212 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
3213 llvm::StringRef intrinsicName;
3214 switch (builtinID) {
3215 default:
3216 llvm_unreachable("Unexpected builtin");
3217 case X86::BI__builtin_ia32_aesencwide128kl_u8:
3218 intrinsicName = "x86.aesencwide128kl";
3219 break;
3220 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
3221 intrinsicName = "x86.aesdecwide128kl";
3222 break;
3223 case X86::BI__builtin_ia32_aesencwide256kl_u8:
3224 intrinsicName = "x86.aesencwide256kl";
3225 break;
3226 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
3227 intrinsicName = "x86.aesdecwide256kl";
3228 break;
3229 }
3230
3231 return emitX86Aeswide(builder, loc: getLoc(srcLoc: expr->getExprLoc()), intrinsicName,
3232 retType: convertType(t: expr->getType()), ops);
3233 }
3234 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
3235 case X86::BI__builtin_ia32_vfmaddcph512_mask:
3236 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
3237 case X86::BI__builtin_ia32_vfmaddcsh_round_mask:
3238 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
3239 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3:
3240 case X86::BI__builtin_ia32_prefetchi:
3241 cgm.errorNYI(expr->getSourceRange(),
3242 std::string("unimplemented X86 builtin call: ") +
3243 getContext().BuiltinInfo.getName(ID: builtinID));
3244 return mlir::Value{};
3245 }
3246}
3247