1//===---- CIRGenBuiltinAArch64.cpp - Emit CIR for AArch64 builtins --------===//
2// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3// See https://llvm.org/LICENSE.txt for license information.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// This contains code to emit ARM64 Builtin calls as CIR or a function call
9// to be later resolved.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenBuilder.h"
14#include "CIRGenFunction.h"
15#include "clang/Basic/AArch64CodeGenUtils.h"
16#include "clang/Basic/TargetBuiltins.h"
17#include "clang/CIR/Dialect/IR/CIRTypes.h"
18#include "clang/CIR/MissingFeatures.h"
19#include "clang/CodeGenUtils/TargetUtils.h"
20
21// TODO(cir): once all builtins are covered, decide whether we still
22// need to use LLVM intrinsics or if there's a better approach to follow. Right
23// now the intrinsics are reused to make it convenient to encode all thousands
24// of them and passing down to LLVM lowering.
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/IntrinsicsAArch64.h"
27
28#include "mlir/IR/BuiltinTypes.h"
29#include "mlir/IR/Value.h"
30#include "clang/AST/GlobalDecl.h"
31#include "clang/Basic/Builtins.h"
32
33using namespace clang;
34using namespace clang::CIRGen;
35using namespace llvm;
36using namespace clang::aarch64;
37
38// Generate vscale * scalingFactor
39static mlir::Value genVscaleTimesFactor(mlir::Location loc,
40 CIRGenBuilderTy &builder,
41 mlir::Type cirTy,
42 int32_t scalingFactor) {
43 mlir::Value vscale = builder.emitIntrinsicCallOp(loc, str: "vscale", resTy: cirTy);
44 return builder.createNUWAMul(loc, lhs: vscale,
45 rhs: builder.getUInt64(c: scalingFactor, loc));
46}
47
48#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
49 {SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
50
51#define SVEMAP2(NameBase, TypeModifier) \
52 {SVE::BI__builtin_sve_##NameBase, 0, TypeModifier}
53static const AArch64SVEAndSMEVectorIntrinsicInfo aarch64SVEIntrinsicMap[] = {
54#define GET_SVE_LLVM_INTRINSIC_MAP
55#include "clang/Basic/arm_sve_builtin_cg.inc"
56#undef GET_SVE_LLVM_INTRINSIC_MAP
57};
58
59static bool aarch64SIMDIntrinsicsProvenSorted = false;
60static bool aarch64SISDIntrinsicsProvenSorted = false;
61static bool aarch64SVEIntrinsicsProvenSorted = false;
62
63// Check if Builtin `builtinId` is present in `intrinsicMap`. If yes, returns
64// the corresponding info struct.
65template <typename IntrinsicInfo>
66static const IntrinsicInfo *
67findARMVectorIntrinsicInMap(ArrayRef<IntrinsicInfo> intrinsicMap,
68 unsigned builtinID, bool &mapProvenSorted) {
69
70#ifndef NDEBUG
71 if (!mapProvenSorted) {
72 assert(llvm::is_sorted(intrinsicMap));
73 mapProvenSorted = true;
74 }
75#endif
76
77 const IntrinsicInfo *info = llvm::lower_bound(intrinsicMap, builtinID);
78
79 if (info != intrinsicMap.end() && info->BuiltinID == builtinID)
80 return info;
81
82 return nullptr;
83}
84
85//===----------------------------------------------------------------------===//
86// Generic helpers
87//===----------------------------------------------------------------------===//
88static llvm::StringRef getLLVMIntrNameNoPrefix(llvm::Intrinsic::ID intrID) {
89 llvm::StringRef llvmIntrName = llvm::Intrinsic::getBaseName(id: intrID);
90 assert(llvmIntrName.starts_with("llvm.") && "Not an LLVM intrinsic!");
91 return llvmIntrName.drop_front(/*strlen("llvm.")=*/N: 5);
92}
93
94static cir::VectorType getFloatNeonType(CIRGenFunction &cgf,
95 NeonTypeFlags intTypeFlags) {
96 int isQuad = intTypeFlags.isQuad();
97 switch (intTypeFlags.getEltType()) {
98 case NeonTypeFlags::Int16:
99 return cir::VectorType::getFixed(elementType: cgf.fP16Ty, size: (4ULL << isQuad));
100 case NeonTypeFlags::Int32:
101 return cir::VectorType::getFixed(elementType: cgf.floatTy, size: (2ULL << isQuad));
102 case NeonTypeFlags::Int64:
103 return cir::VectorType::getFixed(elementType: cgf.doubleTy, size: (1ULL << isQuad));
104 default:
105 llvm_unreachable("Type can't be converted to floating-point!");
106 }
107}
108
109static int64_t getIntValueFromConstOp(mlir::Value val) {
110 return val.getDefiningOp<cir::ConstantOp>().getIntValue().getSExtValue();
111}
112
113/// Build a constant shift amount vector of `vecTy` to shift a vector
114/// Here `shiftVal` is a constant integer that will be broadcast into a
115/// a const vector of `vecTy` which is the return value of this function
116/// If `neg` is true, the shift amount is negated before splatting (used
117/// when encoding a right shift as a left shift by a negative amount for
118/// intrinsics like aarch64.neon.{s,u}rshl).
119static mlir::Value emitNeonShiftVector(CIRGenBuilderTy &builder,
120 mlir::Value shiftVal,
121 cir::VectorType vecTy,
122 mlir::Location loc, bool neg) {
123 if (neg) {
124 int64_t shiftAmt = -getIntValueFromConstOp(val: shiftVal);
125 shiftVal = builder.getConstantInt(loc, ty: vecTy.getElementType(), value: shiftAmt);
126 }
127 mlir::Type eltTy = vecTy.getElementType();
128 if (shiftVal.getType() != eltTy) {
129 shiftVal = builder.createIntCast(src: shiftVal, newTy: eltTy);
130 }
131 return cir::VecSplatOp::create(builder, location: loc, result: vecTy, value: shiftVal);
132}
133
134// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
135template <typename Operation>
136static mlir::Value
137emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder,
138 llvm::SmallVector<mlir::Type> argTypes,
139 llvm::SmallVectorImpl<mlir::Value> &args,
140 std::optional<llvm::StringRef> intrinsicName,
141 mlir::Type funcResTy, mlir::Location loc,
142 bool isConstrainedFPIntrinsic = false, unsigned shift = 0,
143 bool rightshift = false) {
144 // TODO(cir): Consider removing the following unreachable when we have
145 // emitConstrainedFPCall feature implemented
146 assert(!cir::MissingFeatures::emitConstrainedFPCall());
147 if (isConstrainedFPIntrinsic)
148 cgm.errorNYI(loc, feature: std::string("constrained FP intrinsic"));
149
150 for (unsigned j = 0; j < argTypes.size(); ++j) {
151 if (isConstrainedFPIntrinsic) {
152 assert(!cir::MissingFeatures::emitConstrainedFPCall());
153 }
154 if (shift > 0 && shift == j) {
155 args[j] = emitNeonShiftVector(builder, shiftVal: args[j],
156 vecTy: mlir::cast<cir::VectorType>(Val&: argTypes[j]),
157 loc, neg: rightshift);
158 } else {
159 args[j] = builder.createBitcast(src: args[j], newTy: argTypes[j]);
160 }
161 }
162 if (isConstrainedFPIntrinsic) {
163 assert(!cir::MissingFeatures::emitConstrainedFPCall());
164 return nullptr;
165 }
166 if constexpr (std::is_same_v<Operation, cir::LLVMIntrinsicCallOp>) {
167 return Operation::create(builder, loc,
168 builder.getStringAttr(bytes: intrinsicName.value()),
169 funcResTy, args)
170 .getResult();
171 }
172 if constexpr (std::is_same_v<Operation, cir::FMAOp>) {
173 assert(args.size() == 3 && "fma expects three operands");
174 return Operation::create(builder, loc, funcResTy, args[0], args[1], args[2],
175 builder.getConstrainedFPAttr(),
176 builder.getFastMathFlagsAttr())
177 .getResult();
178 }
179 if constexpr (std::is_same_v<Operation, cir::SqrtOp>) {
180 assert(args.size() == 1 && "sqrt expects one operand");
181 return Operation::create(builder, loc, funcResTy, args[0],
182 builder.getConstrainedFPAttr())
183 .getResult();
184 }
185 if constexpr (!std::is_same_v<Operation, cir::LLVMIntrinsicCallOp> &&
186 !std::is_same_v<Operation, cir::FMAOp> &&
187 !std::is_same_v<Operation, cir::SqrtOp>)
188 return Operation::create(
189 builder, loc, mlir::TypeRange{funcResTy}, args,
190 cir::getDefaultProperties<Operation>(builder.getContext()))
191 .getResult();
192}
193
194// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
195static mlir::Value emitNeonCall(CIRGenModule &cgm, CIRGenBuilderTy &builder,
196 llvm::SmallVector<mlir::Type> argTypes,
197 llvm::SmallVectorImpl<mlir::Value> &args,
198 llvm::StringRef intrinsicName,
199 mlir::Type funcResTy, mlir::Location loc,
200 bool isConstrainedFPIntrinsic = false,
201 unsigned shift = 0, bool rightshift = false) {
202 return emitNeonCallToOp<cir::LLVMIntrinsicCallOp>(
203 cgm, builder, argTypes: std::move(argTypes), args, intrinsicName, funcResTy, loc,
204 isConstrainedFPIntrinsic, shift, rightshift);
205}
206
207// Computes the input vector type for a NEON pairwise widening operation (e.g.
208// vpaddl/vpadal). Given a result vector type, it derives the corresponding
209// input type by halving the element bit width and doubling the number of lanes,
210// while setting the signedness based on usgn.
211static cir::VectorType getNeonPairwiseWidenInputType(cir::VectorType resType,
212 bool usgn) {
213 mlir::Type elemTy = resType.getElementType();
214 uint64_t resLanes = resType.getFixedSize();
215 auto intTy = mlir::dyn_cast<cir::IntType>(Val&: elemTy);
216 assert(intTy && "vpaddl result type must be an integer vector");
217
218 unsigned resWidth = intTy.getWidth();
219 assert((resWidth == 16 || resWidth == 32 || resWidth == 64) &&
220 "unexpected vpaddl result element width");
221
222 unsigned argWidth = resWidth / 2;
223 unsigned argLanes = resLanes * 2;
224 cir::VectorType result = cir::VectorType::getFixed(
225 elementType: cir::IntType::get(context: resType.getContext(), width: argWidth, /* is_signed */ !usgn),
226 size: argLanes);
227 return result;
228}
229
230// Derive the LLVM intrinsic's per-operand argument types and its result
231// type for use when emitting the intrinsic call.
232//
233// `modifier` is the TypeModifier bitmask from `ARMNeonVectorIntrinsicInfo`
234// (callers pass `info.TypeModifier`; see AArch64CodeGenUtils.h). It encodes
235// how the intrinsic's argument and return types relate to the builtin's
236// scalar types. For SISD builtins the key flags are:
237// - VectorizeArgTypes: wrap each arg type into a fixed-width vector
238// - Use64BitVectors / Use128BitVectors: choose the vector width
239// (when neither is set the vector has 1 element)
240// - AddRetType / VectorizeRetType: analogous flags for the return type
241//
242// ARM.cpp lets LLVM resolve the intrinsic's signature (via
243// `CGM.getIntrinsic`) and then walks the resolved Function* formal
244// parameter types. CIR has no LLVMContext here, so we derive the same
245// argument/result types directly from the Clang operand types.
246static std::pair<mlir::Type, llvm::SmallVector<mlir::Type>>
247deriveNeonSISDIntrinsicOperandTypes(CIRGenFunction &cgf, unsigned modifier,
248 mlir::Type arg0Ty, mlir::Type resultTy,
249 llvm::ArrayRef<mlir::Value> ops,
250 unsigned iceArguments) {
251 int vectorSize = 0;
252 if (modifier & Use64BitVectors)
253 vectorSize = 64;
254 else if (modifier & Use128BitVectors)
255 vectorSize = 128;
256
257 auto wrapAsVector = [&](mlir::Type ty) -> cir::VectorType {
258 unsigned bits = cgf.cgm.getDataLayout().getTypeSizeInBits(ty);
259 unsigned elts = vectorSize ? vectorSize / bits : 1;
260 return cir::VectorType::getFixed(elementType: ty, size: elts);
261 };
262
263 // Determine the vectorized data type.
264 cir::VectorType vecArgTy;
265 if (modifier & VectorizeArgTypes)
266 vecArgTy = wrapAsVector(arg0Ty);
267
268 // Determine the intrinsic result type: `VectorizeRetType` returns a
269 // vector; otherwise, if data args are vectorized and `AddRetType` is
270 // unset, use a vector return with the same shape as those args.
271 mlir::Type funcResTy = resultTy;
272 if (modifier & VectorizeRetType)
273 funcResTy = wrapAsVector(resultTy);
274 else if (vecArgTy && !(modifier & AddRetType))
275 funcResTy = wrapAsVector(resultTy);
276
277 // AdvSIMD_2Arg_Scalar_Narrow_Intrinsic in IntrinsicsAArch64.td models
278 // operand 0 as LLVMExtendedType<0>: the result vector's lane count with
279 // double-width elements. Reconstruct that LLVM operand type from the Clang
280 // builtin's scalar argument type.
281 if (modifier & ArgAsWidenedRetType) {
282 auto resVecTy = mlir::dyn_cast<cir::VectorType>(Val&: funcResTy);
283 assert(resVecTy &&
284 "SISD LLVM argument reconstruction requires a vector result");
285 vecArgTy = cir::VectorType::get(element_type: arg0Ty, size: resVecTy.getSize());
286 }
287
288 // True if `ty` is arg0's type, or an integer of the same width that only
289 // differs in signedness. vsqadd/vuqadd mix the two on purpose: vsqaddb_u8
290 // takes a uint8_t and an int8_t, but both become the same vector type.
291 auto matchesArg0Ty = [&](mlir::Type ty) {
292 if (ty == arg0Ty)
293 return true;
294 auto intTy = mlir::dyn_cast<cir::IntType>(Val&: ty);
295 auto arg0IntTy = mlir::dyn_cast<cir::IntType>(Val&: arg0Ty);
296 return intTy && arg0IntTy && intTy.getWidth() == arg0IntTy.getWidth();
297 };
298
299 // `vecArgTy` is populated by `VectorizeArgTypes` or
300 // `ArgAsWidenedRetType`. When set, wrap every non-immediate data operand
301 // that has the same scalar type as arg0. Checking the ICE bitmap prevents
302 // an i32 immediate from being vectorized when it has the same type as a
303 // data operand (e.g. vqshrns_n_s32). `VectorizeImmArg` marks the entries
304 // whose trailing immediate must be widened as well.
305 const bool widenTrailingImm = vecArgTy && ops.size() == 2 &&
306 (modifier & VectorizeImmArg) &&
307 (iceArguments & (1U << 1));
308 llvm::SmallVector<mlir::Type> argTypes;
309 argTypes.reserve(N: ops.size());
310 for (unsigned i = 0, e = ops.size(); i != e; ++i) {
311 bool isImmediate = iceArguments & (1U << i);
312 bool asVector =
313 vecArgTy && ((!isImmediate && matchesArg0Ty(ops[i].getType())) ||
314 (widenTrailingImm && i == 1));
315 argTypes.push_back(Elt: asVector ? mlir::Type(vecArgTy) : ops[i].getType());
316 }
317
318 return {funcResTy, std::move(argTypes)};
319}
320
321// Source-operand vector type for a common NEON binary intrinsic: the
322// double-element-width form of `vTy` when `ArgAsWidenedRetType` is set (e.g.
323// vraddhn), otherwise `vTy`.
324static cir::VectorType deriveNeonBinaryArgType(CIRGenBuilderTy &builder,
325 unsigned modifier,
326 cir::VectorType vTy) {
327 if (modifier & ArgAsWidenedRetType)
328 return builder.getExtendedOrTruncatedElementVectorType(vt: vTy,
329 /*isExtended=*/true);
330 return vTy;
331}
332
333/// Create a vector from an input scalar argument, usually for a NEON SISD
334/// intrinsic call. Insert the argument into lane 0 of a poison vector.
335static void vecExtendIntValue(CIRGenFunction &cgf, cir::VectorType argVTy,
336 mlir::Value &arg, mlir::Location loc) {
337 CIRGenBuilderTy &builder = cgf.getBuilder();
338 // TODO: Support floating-point scalar arguments when a SISD intrinsic
339 // requires scalar-to-vector adaptation; current users are integer-only.
340 cir::IntType eltTy = mlir::dyn_cast<cir::IntType>(Val: argVTy.getElementType());
341 assert(mlir::isa<cir::IntType>(arg.getType()) && eltTy);
342 // Cast the scalar data operand to the vector element type before inserting
343 // it into lane 0.
344 arg = builder.createIntCast(src: arg, newTy: eltTy);
345 mlir::Value zero = builder.getConstInt(loc, t: cgf.sizeTy, c: 0);
346 mlir::Value poison = builder.getConstant(loc, attr: cir::PoisonAttr::get(type: argVTy));
347 arg = cir::VecInsertOp::create(builder, location: loc, vec: poison, value: arg, index: zero);
348}
349
350static mlir::Value
351emitCommonNeonSISDBuiltinExpr(CIRGenFunction &cgf,
352 const ARMNeonVectorIntrinsicInfo &info,
353 llvm::SmallVectorImpl<mlir::Value> &ops,
354 const CallExpr *expr, unsigned iceArguments) {
355 assert(info.LLVMIntrinsic && "Generic code assumes a valid intrinsic");
356
357 switch (info.BuiltinID) {
358 case NEON::BI__builtin_neon_vcled_s64:
359 case NEON::BI__builtin_neon_vcled_u64:
360 case NEON::BI__builtin_neon_vcles_f32:
361 case NEON::BI__builtin_neon_vcled_f64:
362 case NEON::BI__builtin_neon_vcltd_s64:
363 case NEON::BI__builtin_neon_vcltd_u64:
364 case NEON::BI__builtin_neon_vclts_f32:
365 case NEON::BI__builtin_neon_vcltd_f64:
366 case NEON::BI__builtin_neon_vcales_f32:
367 case NEON::BI__builtin_neon_vcaled_f64:
368 case NEON::BI__builtin_neon_vcalts_f32:
369 case NEON::BI__builtin_neon_vcaltd_f64:
370 cgf.cgm.errorNYI(expr->getSourceRange(),
371 std::string("unimplemented AArch64 builtin call: ") +
372 cgf.getContext().BuiltinInfo.getName(ID: info.BuiltinID));
373 break;
374 }
375
376 unsigned intr = info.LLVMIntrinsic;
377
378 // Use fptosi.sat/fptoui.sat unless under strict FP.
379 assert(!cir::MissingFeatures::emitConstrainedFPCall());
380 if (intr == Intrinsic::aarch64_neon_fcvtzs)
381 intr = Intrinsic::fptosi_sat;
382 else if (intr == Intrinsic::aarch64_neon_fcvtzu)
383 intr = Intrinsic::fptoui_sat;
384
385 llvm::StringRef llvmIntrName =
386 getLLVMIntrNameNoPrefix(intrID: static_cast<llvm::Intrinsic::ID>(intr));
387 mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc());
388
389 // The switch stmt is intended to help catch NYI cases and will be removed
390 // once the CIR implementation is complete. Avoid adding specialized
391 // code in cases - that should only be required for a handful of examples.
392 switch (info.BuiltinID) {
393 default:
394 cgf.cgm.errorNYI(expr->getSourceRange(),
395 std::string("unimplemented AArch64 builtin call: ") +
396 cgf.getContext().BuiltinInfo.getName(ID: info.BuiltinID));
397 break;
398 case NEON::BI__builtin_neon_vminv_s8:
399 case NEON::BI__builtin_neon_vminvq_s8:
400 case NEON::BI__builtin_neon_vminv_s16:
401 case NEON::BI__builtin_neon_vminvq_s16:
402 case NEON::BI__builtin_neon_vminv_s32:
403 case NEON::BI__builtin_neon_vminvq_s32:
404 case NEON::BI__builtin_neon_vminv_u8:
405 case NEON::BI__builtin_neon_vminvq_u8:
406 case NEON::BI__builtin_neon_vminv_u16:
407 case NEON::BI__builtin_neon_vminvq_u16:
408 case NEON::BI__builtin_neon_vminv_u32:
409 case NEON::BI__builtin_neon_vminvq_u32:
410 case NEON::BI__builtin_neon_vminv_f32:
411 case NEON::BI__builtin_neon_vminvq_f32:
412 case NEON::BI__builtin_neon_vminvq_f64:
413 case NEON::BI__builtin_neon_vminnmv_f32:
414 case NEON::BI__builtin_neon_vminnmvq_f32:
415 case NEON::BI__builtin_neon_vminnmvq_f64:
416 case NEON::BI__builtin_neon_vabdd_f64:
417 case NEON::BI__builtin_neon_vabds_f32:
418 case NEON::BI__builtin_neon_vshld_s64:
419 case NEON::BI__builtin_neon_vshld_u64:
420 case NEON::BI__builtin_neon_vpmins_f32:
421 case NEON::BI__builtin_neon_vpminqd_f64:
422 case NEON::BI__builtin_neon_vpminnms_f32:
423 case NEON::BI__builtin_neon_vpminnmqd_f64:
424 case NEON::BI__builtin_neon_vcvtd_s32_f64:
425 case NEON::BI__builtin_neon_vcvtd_s64_f64:
426 case NEON::BI__builtin_neon_vcvtd_u64_f64:
427 case NEON::BI__builtin_neon_vcvtd_u32_f64:
428 case NEON::BI__builtin_neon_vcvts_n_f32_s32:
429 case NEON::BI__builtin_neon_vcvts_n_f32_u32:
430 case NEON::BI__builtin_neon_vcvts_n_s32_f32:
431 case NEON::BI__builtin_neon_vcvts_n_u32_f32:
432 case NEON::BI__builtin_neon_vcvtd_n_f64_s64:
433 case NEON::BI__builtin_neon_vcvtd_n_f64_u64:
434 case NEON::BI__builtin_neon_vcvtd_n_s64_f64:
435 case NEON::BI__builtin_neon_vcvtd_n_u64_f64:
436 case NEON::BI__builtin_neon_vcvts_s32_f32:
437 case NEON::BI__builtin_neon_vcvts_s64_f32:
438 case NEON::BI__builtin_neon_vcvts_u32_f32:
439 case NEON::BI__builtin_neon_vcvts_u64_f32:
440 case NEON::BI__builtin_neon_vcvtns_s32_f32:
441 case NEON::BI__builtin_neon_vcvtns_s64_f32:
442 case NEON::BI__builtin_neon_vcvtns_u32_f32:
443 case NEON::BI__builtin_neon_vcvtns_u64_f32:
444 case NEON::BI__builtin_neon_vcvtnd_s32_f64:
445 case NEON::BI__builtin_neon_vcvtnd_s64_f64:
446 case NEON::BI__builtin_neon_vcvtnd_u32_f64:
447 case NEON::BI__builtin_neon_vcvtnd_u64_f64:
448 case NEON::BI__builtin_neon_vcvtms_s32_f32:
449 case NEON::BI__builtin_neon_vcvtms_s64_f32:
450 case NEON::BI__builtin_neon_vcvtms_u32_f32:
451 case NEON::BI__builtin_neon_vcvtms_u64_f32:
452 case NEON::BI__builtin_neon_vcvtmd_s32_f64:
453 case NEON::BI__builtin_neon_vcvtmd_u32_f64:
454 case NEON::BI__builtin_neon_vcvtmd_s64_f64:
455 case NEON::BI__builtin_neon_vcvtmd_u64_f64:
456 case NEON::BI__builtin_neon_vcvtps_s32_f32:
457 case NEON::BI__builtin_neon_vcvtps_s64_f32:
458 case NEON::BI__builtin_neon_vcvtps_u32_f32:
459 case NEON::BI__builtin_neon_vcvtps_u64_f32:
460 case NEON::BI__builtin_neon_vcvtpd_s32_f64:
461 case NEON::BI__builtin_neon_vcvtpd_u32_f64:
462 case NEON::BI__builtin_neon_vcvtpd_s64_f64:
463 case NEON::BI__builtin_neon_vcvtpd_u64_f64:
464 case NEON::BI__builtin_neon_vcvtas_s32_f32:
465 case NEON::BI__builtin_neon_vcvtas_s64_f32:
466 case NEON::BI__builtin_neon_vcvtas_u32_f32:
467 case NEON::BI__builtin_neon_vcvtas_u64_f32:
468 case NEON::BI__builtin_neon_vcvtad_s32_f64:
469 case NEON::BI__builtin_neon_vcvtad_u32_f64:
470 case NEON::BI__builtin_neon_vcvtad_s64_f64:
471 case NEON::BI__builtin_neon_vcvtad_u64_f64:
472 case NEON::BI__builtin_neon_vcvtxd_f32_f64:
473 case NEON::BI__builtin_neon_vaddlv_s32:
474 case NEON::BI__builtin_neon_vaddlv_u32:
475 case NEON::BI__builtin_neon_vaddlvq_s32:
476 case NEON::BI__builtin_neon_vaddlvq_u32:
477 case NEON::BI__builtin_neon_vaddv_s8:
478 case NEON::BI__builtin_neon_vaddv_s16:
479 case NEON::BI__builtin_neon_vaddv_s32:
480 case NEON::BI__builtin_neon_vaddv_u8:
481 case NEON::BI__builtin_neon_vaddv_u16:
482 case NEON::BI__builtin_neon_vaddv_u32:
483 case NEON::BI__builtin_neon_vaddv_f32:
484 case NEON::BI__builtin_neon_vaddvq_s8:
485 case NEON::BI__builtin_neon_vaddvq_s16:
486 case NEON::BI__builtin_neon_vaddvq_s32:
487 case NEON::BI__builtin_neon_vaddvq_s64:
488 case NEON::BI__builtin_neon_vaddvq_u8:
489 case NEON::BI__builtin_neon_vaddvq_u16:
490 case NEON::BI__builtin_neon_vaddvq_u32:
491 case NEON::BI__builtin_neon_vaddvq_u64:
492 case NEON::BI__builtin_neon_vaddvq_f32:
493 case NEON::BI__builtin_neon_vaddvq_f64:
494 case NEON::BI__builtin_neon_vabdh_f16:
495 case NEON::BI__builtin_neon_vrecpeh_f16:
496 case NEON::BI__builtin_neon_vrecpxh_f16:
497 case NEON::BI__builtin_neon_vrsqrteh_f16:
498 case NEON::BI__builtin_neon_vrsqrtsh_f16:
499 case NEON::BI__builtin_neon_vmaxv_s8:
500 case NEON::BI__builtin_neon_vmaxvq_s8:
501 case NEON::BI__builtin_neon_vmaxv_s16:
502 case NEON::BI__builtin_neon_vmaxvq_s16:
503 case NEON::BI__builtin_neon_vmaxv_s32:
504 case NEON::BI__builtin_neon_vmaxvq_s32:
505 case NEON::BI__builtin_neon_vmaxv_u8:
506 case NEON::BI__builtin_neon_vmaxvq_u8:
507 case NEON::BI__builtin_neon_vmaxv_u16:
508 case NEON::BI__builtin_neon_vmaxvq_u16:
509 case NEON::BI__builtin_neon_vmaxv_u32:
510 case NEON::BI__builtin_neon_vmaxvq_u32:
511 case NEON::BI__builtin_neon_vmaxv_f32:
512 case NEON::BI__builtin_neon_vmaxvq_f32:
513 case NEON::BI__builtin_neon_vmaxvq_f64:
514 case NEON::BI__builtin_neon_vqrshrund_n_s64:
515 case NEON::BI__builtin_neon_vqrshrnd_n_s64:
516 case NEON::BI__builtin_neon_vqrshrnd_n_u64:
517 case NEON::BI__builtin_neon_vqshrund_n_s64:
518 case NEON::BI__builtin_neon_vqshrnd_n_s64:
519 case NEON::BI__builtin_neon_vqshrnd_n_u64:
520 case NEON::BI__builtin_neon_vmaxnmv_f32:
521 case NEON::BI__builtin_neon_vmaxnmvq_f32:
522 case NEON::BI__builtin_neon_vmaxnmvq_f64:
523 case NEON::BI__builtin_neon_vsrid_n_s64:
524 case NEON::BI__builtin_neon_vsrid_n_u64:
525 case NEON::BI__builtin_neon_vslid_n_s64:
526 case NEON::BI__builtin_neon_vslid_n_u64:
527 case NEON::BI__builtin_neon_vpmaxs_f32:
528 case NEON::BI__builtin_neon_vpmaxqd_f64:
529 case NEON::BI__builtin_neon_vpmaxnms_f32:
530 case NEON::BI__builtin_neon_vpmaxnmqd_f64:
531 case NEON::BI__builtin_neon_vmulxh_f16:
532 case NEON::BI__builtin_neon_vqshrunh_n_s16:
533 case NEON::BI__builtin_neon_vqshruns_n_s32:
534 case NEON::BI__builtin_neon_vqshrnh_n_s16:
535 case NEON::BI__builtin_neon_vqshrns_n_s32:
536 case NEON::BI__builtin_neon_vqshrnh_n_u16:
537 case NEON::BI__builtin_neon_vqshrns_n_u32:
538 case NEON::BI__builtin_neon_vqsubb_s8:
539 case NEON::BI__builtin_neon_vqsubb_u8:
540 case NEON::BI__builtin_neon_vqsubh_s16:
541 case NEON::BI__builtin_neon_vqsubh_u16:
542 case NEON::BI__builtin_neon_vqsubs_s32:
543 case NEON::BI__builtin_neon_vqsubs_u32:
544 case NEON::BI__builtin_neon_vqsubd_s64:
545 case NEON::BI__builtin_neon_vqsubd_u64:
546 case NEON::BI__builtin_neon_vqaddb_s8:
547 case NEON::BI__builtin_neon_vqaddb_u8:
548 case NEON::BI__builtin_neon_vqaddh_s16:
549 case NEON::BI__builtin_neon_vqaddh_u16:
550 case NEON::BI__builtin_neon_vqadds_s32:
551 case NEON::BI__builtin_neon_vqadds_u32:
552 case NEON::BI__builtin_neon_vqaddd_s64:
553 case NEON::BI__builtin_neon_vqaddd_u64:
554 case NEON::BI__builtin_neon_vsqaddb_u8:
555 case NEON::BI__builtin_neon_vsqaddh_u16:
556 case NEON::BI__builtin_neon_vsqadds_u32:
557 case NEON::BI__builtin_neon_vsqaddd_u64:
558 case NEON::BI__builtin_neon_vuqaddb_s8:
559 case NEON::BI__builtin_neon_vuqaddh_s16:
560 case NEON::BI__builtin_neon_vuqadds_s32:
561 case NEON::BI__builtin_neon_vuqaddd_s64:
562 case NEON::BI__builtin_neon_vqrshrns_n_s32:
563 case NEON::BI__builtin_neon_vqrshrns_n_u32:
564 case NEON::BI__builtin_neon_vqrshrnh_n_s16:
565 case NEON::BI__builtin_neon_vqrshrnh_n_u16:
566 case NEON::BI__builtin_neon_vqrshruns_n_s32:
567 case NEON::BI__builtin_neon_vqrshrunh_n_s16:
568 case NEON::BI__builtin_neon_vqshlb_s8:
569 case NEON::BI__builtin_neon_vqshlb_u8:
570 case NEON::BI__builtin_neon_vqshlh_s16:
571 case NEON::BI__builtin_neon_vqshlh_u16:
572 case NEON::BI__builtin_neon_vqshls_s32:
573 case NEON::BI__builtin_neon_vqshls_u32:
574 case NEON::BI__builtin_neon_vqshld_s64:
575 case NEON::BI__builtin_neon_vqshld_u64:
576 case NEON::BI__builtin_neon_vqshlb_n_s8:
577 case NEON::BI__builtin_neon_vqshlb_n_u8:
578 case NEON::BI__builtin_neon_vqshlh_n_s16:
579 case NEON::BI__builtin_neon_vqshlh_n_u16:
580 case NEON::BI__builtin_neon_vqshls_n_s32:
581 case NEON::BI__builtin_neon_vqshls_n_u32:
582 case NEON::BI__builtin_neon_vqshlub_n_s8:
583 case NEON::BI__builtin_neon_vqshluh_n_s16:
584 case NEON::BI__builtin_neon_vqshlus_n_s32:
585 break;
586 }
587
588 CIRGenBuilderTy &builder = cgf.getBuilder();
589 mlir::Type arg0Ty = cgf.convertType(t: expr->getArg(Arg: 0)->getType());
590 mlir::Type resultTy = cgf.convertType(t: expr->getType());
591
592 // Derive per-operand argument types and the result type from the
593 // TypeModifier flags. `emitNeonCall` takes care of per-operand
594 // bitcasts to `argTypes`.
595 auto [funcResTy, argTypes] = deriveNeonSISDIntrinsicOperandTypes(
596 cgf, modifier: info.TypeModifier, arg0Ty, resultTy, ops, iceArguments);
597
598 assert(argTypes.size() == ops.size());
599 for (unsigned i = 0, e = ops.size(); i != e; ++i) {
600 if (cgf.cgm.getDataLayout().getTypeSizeInBits(ty: ops[i].getType()) ==
601 cgf.cgm.getDataLayout().getTypeSizeInBits(ty: argTypes[i]))
602 continue;
603
604 auto argVecTy = mlir::dyn_cast<cir::VectorType>(Val&: argTypes[i]);
605 assert(argVecTy && !mlir::isa<cir::VectorType>(ops[i].getType()) &&
606 "expecting vector LLVM intrinsic type and scalar Clang builtin "
607 "type");
608
609 vecExtendIntValue(cgf, argVTy: argVecTy, arg&: ops[i], loc);
610 }
611
612 mlir::Value result = emitNeonCall(cgm&: cgf.cgm, builder, argTypes: std::move(argTypes), args&: ops,
613 intrinsicName: llvmIntrName, funcResTy, loc);
614
615 if (cgf.cgm.getDataLayout().getTypeSizeInBits(ty: resultTy) <
616 cgf.cgm.getDataLayout().getTypeSizeInBits(ty: funcResTy)) {
617
618 assert(mlir::isa<cir::VectorType>(result.getType()));
619 return cir::VecExtractOp::create(builder, location: loc, vec: result,
620 index: builder.getConstInt(loc, t: cgf.sizeTy, c: 0));
621 }
622
623 return builder.createBitcast(loc, src: result, newTy: resultTy);
624}
625
626//===----------------------------------------------------------------------===//
627// Emit-helpers
628//===----------------------------------------------------------------------===//
629template <typename CmpKindTy>
630static mlir::Value
631emitAArch64CompareBuiltinExpr(CIRGenFunction &cgf, CIRGenBuilderTy &builder,
632 mlir::Location loc, mlir::Value src,
633 mlir::Type retTy, const CmpKindTy kind) {
634
635 bool scalarCmp = !isa<cir::VectorType>(Val: src.getType());
636 if (!scalarCmp) {
637 assert(!cast<cir::VectorType>(retTy).isScalable() &&
638 "This is only intended for fixed-width vectors");
639 // Vector types are cast to i8 vectors. Recover original type.
640 src = builder.createBitcast(src, newTy: retTy);
641 }
642
643 mlir::Value zero = builder.getNullValue(ty: src.getType(), loc);
644
645 if (!scalarCmp) {
646 if constexpr (std::is_same<CmpKindTy, cir::FCmpOpKind>::value)
647 return builder.createVecFCompare(loc, kind, src, zero);
648 else
649 return builder.createVecCompare(loc, kind, src, zero);
650 }
651
652 // FIXME FCmpOpKind is only supported for vector comparisons now, modify the
653 // if when scalar comparisons are supported too;
654 if constexpr (std::is_same<CmpKindTy, cir::CmpOpKind>::value) {
655 // For scalars, cast !cir.bool to !cir.int<s, 1> so that the compare
656 // result is sign- rather zero-extended when casting to the output
657 // retType.
658 mlir::Value cmp = builder.createCast(
659 loc, cir::CastKind::bool_to_int,
660 builder.createCompare(loc, kind, lhs: src, rhs: zero), builder.getSIntNTy(n: 1));
661
662 return builder.createCast(loc, kind: cir::CastKind::integral, src: cmp, newTy: retTy);
663 }
664
665 llvm_unreachable("Should not end up here");
666 return {};
667}
668
669static cir::VectorType getNeonType(CIRGenFunction *cgf, NeonTypeFlags typeFlags,
670 bool hasLegalHalfType = true,
671 bool v1Ty = false,
672 bool allowBFloatArgsAndRet = true) {
673 int isQuad = typeFlags.isQuad();
674 switch (typeFlags.getEltType()) {
675 case NeonTypeFlags::Int8:
676 case NeonTypeFlags::Poly8:
677 return cir::VectorType::getFixed(elementType: typeFlags.isUnsigned() ? cgf->uInt8Ty
678 : cgf->sInt8Ty,
679 size: v1Ty ? 1 : (8 << isQuad));
680 case NeonTypeFlags::MFloat8:
681 return cir::VectorType::getFixed(elementType: cgf->uInt8Ty, size: v1Ty ? 1 : (8 << isQuad));
682 case NeonTypeFlags::Int16:
683 case NeonTypeFlags::Poly16:
684 return cir::VectorType::getFixed(elementType: typeFlags.isUnsigned() ? cgf->uInt16Ty
685 : cgf->sInt16Ty,
686 size: v1Ty ? 1 : (4 << isQuad));
687 case NeonTypeFlags::BFloat16:
688 if (allowBFloatArgsAndRet)
689 return cir::VectorType::getFixed(elementType: cgf->getCIRGenModule().bFloat16Ty,
690 size: v1Ty ? 1 : (4 << isQuad));
691 return cir::VectorType::getFixed(elementType: cgf->uInt16Ty, size: v1Ty ? 1 : (4 << isQuad));
692 case NeonTypeFlags::Float16:
693 if (hasLegalHalfType)
694 return cir::VectorType::getFixed(elementType: cgf->getCIRGenModule().fP16Ty,
695 size: v1Ty ? 1 : (4 << isQuad));
696 return cir::VectorType::getFixed(elementType: cgf->uInt16Ty, size: v1Ty ? 1 : (4 << isQuad));
697 case NeonTypeFlags::Int32:
698 return cir::VectorType::getFixed(elementType: typeFlags.isUnsigned() ? cgf->uInt32Ty
699 : cgf->sInt32Ty,
700 size: v1Ty ? 1 : (2 << isQuad));
701 case NeonTypeFlags::Int64:
702 case NeonTypeFlags::Poly64:
703 return cir::VectorType::getFixed(elementType: typeFlags.isUnsigned() ? cgf->uInt64Ty
704 : cgf->sInt64Ty,
705 size: v1Ty ? 1 : (1 << isQuad));
706 case NeonTypeFlags::Poly128:
707 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
708 // There is a lot of i128 and f128 API missing.
709 // so we use v16i8 to represent poly128 and get pattern matched.
710 return cir::VectorType::getFixed(elementType: cgf->uInt8Ty, size: 16);
711 case NeonTypeFlags::Float32:
712 return cir::VectorType::getFixed(elementType: cgf->getCIRGenModule().floatTy,
713 size: v1Ty ? 1 : (2 << isQuad));
714 case NeonTypeFlags::Float64:
715 return cir::VectorType::getFixed(elementType: cgf->getCIRGenModule().doubleTy,
716 size: v1Ty ? 1 : (1 << isQuad));
717 }
718 llvm_unreachable("Unknown vector element type!");
719}
720
721static mlir::Value emitNeonSplat(CIRGenBuilderTy &builder, mlir::Location loc,
722 mlir::Value v, mlir::Value lane,
723 unsigned int resEltCnt) {
724 assert(isa<cir::ConstantOp>(lane.getDefiningOp()) &&
725 "lane number is not a constant!");
726 int64_t laneCst = getIntValueFromConstOp(val: lane);
727 llvm::SmallVector<int64_t, 4> shuffleMask(resEltCnt, laneCst);
728 return builder.createVecShuffle(loc, vec1: v, mask: shuffleMask);
729}
730
731/// Flip the signedness of `vecTy`'s element type, keeping the width and
732/// number of lanes the same. Used when a NEON intrinsic takes a shift
733/// amount vector that must be signed (e.g. aarch64.neon.urshl takes a
734/// signed amount even though the data vector is unsigned).
735static cir::VectorType getSignChangedVectorType(CIRGenBuilderTy &builder,
736 cir::VectorType vecTy) {
737 auto elemTy = mlir::cast<cir::IntType>(Val: vecTy.getElementType());
738 elemTy = elemTy.isSigned() ? builder.getUIntNTy(n: elemTy.getWidth())
739 : builder.getSIntNTy(n: elemTy.getWidth());
740 return cir::VectorType::get(element_type: elemTy, size: vecTy.getSize());
741}
742
743static mlir::Value emitCommonNeonShift(CIRGenBuilderTy &builder,
744 mlir::Location loc,
745 cir::VectorType resTy,
746 mlir::Value shifTgt,
747 mlir::Value shiftAmt, bool shiftLeft) {
748 shiftAmt = emitNeonShiftVector(builder, shiftVal: shiftAmt, vecTy: resTy, loc, /*neg=*/false);
749 return cir::ShiftOp::create(builder, location: loc, result: resTy,
750 value: builder.createBitcast(src: shifTgt, newTy: resTy), amount: shiftAmt,
751 isShiftleft: shiftLeft);
752}
753
754// Right-shift a vector by a constant.
755static mlir::Value emitNeonRShiftImm(CIRGenFunction &cgf, mlir::Value shiftVec,
756 mlir::Value shiftVal,
757 cir::VectorType vecTy, bool usgn,
758 mlir::Location loc) {
759 CIRGenBuilderTy &builder = cgf.getBuilder();
760 int64_t shiftAmt = getIntValueFromConstOp(val: shiftVal);
761 int eltSize =
762 cgf.cgm.getDataLayout().getTypeSizeInBits(ty: vecTy.getElementType());
763
764 shiftVec = builder.createBitcast(src: shiftVec, newTy: vecTy);
765 // lshr/ashr are undefined when the shift amount is equal to the vector
766 // element size.
767 if (shiftAmt == eltSize) {
768 if (usgn) {
769 // Right-shifting an unsigned value by its size yields 0.
770 return builder.getZero(loc, ty: vecTy);
771 }
772 // Right-shifting a signed value by its size is equivalent
773 // to a shift of size-1.
774 --shiftAmt;
775 shiftVal = builder.getConstInt(loc, t: vecTy.getElementType(), c: shiftAmt);
776 }
777 return emitCommonNeonShift(builder, loc, resTy: vecTy, shifTgt: shiftVec, shiftAmt: shiftVal,
778 /*shiftLeft=*/false);
779}
780
781static cir::VectorType getIntVecFromVecTy(CIRGenBuilderTy &builder,
782 cir::VectorType vecTy) {
783 if (!cir::isAnyFloatingPointType(type: vecTy.getElementType()))
784 return vecTy;
785
786 if (mlir::isa<cir::SingleType>(Val: vecTy.getElementType()))
787 return cir::VectorType::get(element_type: builder.getSInt32Ty(), size: vecTy.getSize());
788 if (mlir::isa<cir::DoubleType>(Val: vecTy.getElementType()))
789 return cir::VectorType::get(element_type: builder.getSInt64Ty(), size: vecTy.getSize());
790 llvm_unreachable(
791 "Unsupported element type in getVecOfIntTypeWithSameEltWidth");
792}
793
794/// The vld1_xN builtins move their data through an aggregate of N vectors.
795/// Return N, or 0 if `builtinID` is not one of them.
796static unsigned getNeonMultiVecCount(unsigned builtinID) {
797 switch (builtinID) {
798 default:
799 return 0;
800 case NEON::BI__builtin_neon_vld1_x2_v:
801 case NEON::BI__builtin_neon_vld1q_x2_v:
802 return 2;
803 case NEON::BI__builtin_neon_vld1_x3_v:
804 case NEON::BI__builtin_neon_vld1q_x3_v:
805 return 3;
806 case NEON::BI__builtin_neon_vld1_x4_v:
807 case NEON::BI__builtin_neon_vld1q_x4_v:
808 return 4;
809 }
810}
811
812/// Return the anonymous record type `{vecTy, ..., vecTy}` (`numVecs` members)
813/// modelling the struct returned by LLVM's multi-vector NEON load intrinsics.
814static cir::RecordType getNeonMultiVecTy(CIRGenBuilderTy &builder,
815 mlir::Type vecTy, unsigned numVecs) {
816 llvm::SmallVector<mlir::Type> members(numVecs, vecTy);
817 return builder.getAnonRecordTy(members, /*packed=*/false,
818 memberKinds: cir::RecordType::getAllDataKinds(members));
819}
820
821/// Call the multi-vector NEON load intrinsic `llvmIntrinsic` with `args` of
822/// `argTypes` and store the resulting `numVecs`-vector aggregate to the
823/// destination pointer `dest`. Mirrors the `CreateCall` +
824/// `CreateDefaultAlignedStore` pair used for `vld1_xN` in
825/// `EmitCommonNeonBuiltinExpr` and for `vld[234]{,_dup,_lane}` in
826/// `EmitAArch64BuiltinExpr` (ARM.cpp).
827static mlir::Value
828emitNeonMultiVecLoadCall(CIRGenFunction &cgf, mlir::Value dest,
829 llvm::SmallVector<mlir::Type> argTypes,
830 llvm::SmallVectorImpl<mlir::Value> &args,
831 llvm::Intrinsic::ID llvmIntrinsic, mlir::Type ty,
832 unsigned numVecs, mlir::Location loc) {
833 CIRGenBuilderTy &builder = cgf.getBuilder();
834 cir::RecordType recTy = getNeonMultiVecTy(builder, vecTy: ty, numVecs);
835 mlir::Value result =
836 emitNeonCall(cgm&: cgf.getCIRGenModule(), builder, argTypes: std::move(argTypes), args,
837 intrinsicName: getLLVMIntrNameNoPrefix(intrID: llvmIntrinsic), funcResTy: recTy, loc);
838 Address destAddr(builder.createPtrBitcast(src: dest, newPointeeTy: recTy), recTy,
839 clang::CharUnits::fromQuantity(
840 Quantity: cgf.cgm.getDataLayout().getABITypeAlign(ty: recTy)));
841 builder.createStore(loc, val: result, dst: destAddr);
842 return nullptr;
843}
844
845/// Emit a multi-vector NEON load: call `llvmIntrinsic` on the source pointer
846/// `ops[1]` and store the resulting aggregate to the destination pointer
847/// `ops[0]`.
848static mlir::Value emitNeonMultiVecLoad(CIRGenFunction &cgf,
849 llvm::ArrayRef<mlir::Value> ops,
850 unsigned llvmIntrinsic, mlir::Type ty,
851 unsigned numVecs, mlir::Location loc) {
852 llvm::SmallVector<mlir::Value> srcOp = {ops[1]};
853 return emitNeonMultiVecLoadCall(
854 cgf, dest: ops[0], argTypes: {cgf.getBuilder().getVoidPtrTy()}, args&: srcOp,
855 llvmIntrinsic: static_cast<llvm::Intrinsic::ID>(llvmIntrinsic), ty, numVecs, loc);
856}
857
858/// Emit a multi-vector NEON load-lane: call `llvmIntrinsic` on the `numVecs`
859/// vector operands, the lane index, and the source pointer `ops[1]`, then
860/// store the resulting aggregate to the destination pointer `ops[0]`.
861/// `ops` on entry is `[dest, addr, vec0, ..., vec(numVecs-1), lane]`.
862static mlir::Value emitNeonMultiVecLoadLane(CIRGenFunction &cgf,
863 llvm::ArrayRef<mlir::Value> ops,
864 llvm::Intrinsic::ID llvmIntrinsic,
865 mlir::Type ty, unsigned numVecs,
866 mlir::Location loc) {
867 assert(ops.size() == numVecs + 3 && "expected [dest, addr, vecs..., lane]");
868 CIRGenBuilderTy &builder = cgf.getBuilder();
869 // Reorder to [vec0, ..., vec(numVecs-1), lane, addr] for the intrinsic call.
870 llvm::SmallVector<mlir::Value> callOps(ops.begin() + 2, ops.end());
871 callOps.push_back(Elt: ops[1]);
872 // The lane is a Sema-range-checked non-negative constant, so this
873 // sign-extending cast is equivalent to classic CodeGen's CreateZExt.
874 callOps[numVecs] = builder.createIntCast(src: callOps[numVecs], newTy: cgf.sInt64Ty);
875 llvm::SmallVector<mlir::Type> argTypes(numVecs, ty);
876 argTypes.push_back(Elt: cgf.sInt64Ty);
877 argTypes.push_back(Elt: builder.getVoidPtrTy());
878 return emitNeonMultiVecLoadCall(cgf, dest: ops[0], argTypes: std::move(argTypes), args&: callOps,
879 llvmIntrinsic, ty, numVecs, loc);
880}
881
882static mlir::Value emitCommonNeonBuiltinExpr(
883 CIRGenFunction &cgf, unsigned builtinID, unsigned llvmIntrinsic,
884 unsigned altLLVMIntrinsic, const char *nameHint, unsigned modifier,
885 const CallExpr *expr, llvm::SmallVectorImpl<mlir::Value> &ops) {
886 mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc());
887 clang::ASTContext &ctx = cgf.getContext();
888
889 // Extract the trailing immediate argument that encodes the type discriminator
890 // for this overloaded intrinsic.
891 // TODO: Move to the parent code that takes care of argument processing.
892 const clang::Expr *arg = expr->getArg(Arg: expr->getNumArgs() - 1);
893 std::optional<llvm::APSInt> neonTypeConst = arg->getIntegerConstantExpr(Ctx: ctx);
894 if (!neonTypeConst)
895 return nullptr;
896
897 // Determine the type of this overloaded NEON intrinsic.
898 NeonTypeFlags neonType(neonTypeConst->getZExtValue());
899 const bool isUnsigned = neonType.isUnsigned();
900 const bool hasLegalHalfType = cgf.getTarget().hasFastHalfType();
901 const bool usgn = neonType.isUnsigned();
902 const bool isQuad = neonType.isQuad();
903 const bool floating = neonType.isFloatingPoint();
904
905 // The value of allowBFloatArgsAndRet is true for AArch64, but it should
906 // come from ABI info.
907 // TODO(cir): Use ABInfo to extract this information
908 const bool allowBFloatArgsAndRet = cgf.getTarget().hasFastHalfType();
909 // FIXME
910 // getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
911
912 cir::VectorType vTy = getNeonType(cgf: &cgf, typeFlags: neonType, hasLegalHalfType,
913 /*v1Ty=*/false, allowBFloatArgsAndRet);
914 cir::VectorType ty = vTy;
915 if (!ty)
916 return nullptr;
917
918 switch (builtinID) {
919 case NEON::BI__builtin_neon_splat_lane_v:
920 case NEON::BI__builtin_neon_splat_laneq_v:
921 case NEON::BI__builtin_neon_splatq_lane_v:
922 case NEON::BI__builtin_neon_splatq_laneq_v: {
923 uint64_t numElements = vTy.getFixedSize();
924 if (builtinID == NEON::BI__builtin_neon_splatq_lane_v)
925 numElements *= 2;
926 if (builtinID == NEON::BI__builtin_neon_splat_laneq_v)
927 numElements /= 2;
928 ops[0] = cgf.getBuilder().createBitcast(loc, src: ops[0], newTy: vTy);
929 return emitNeonSplat(builder&: cgf.getBuilder(), loc, v: ops[0], lane: ops[1], resEltCnt: numElements);
930 }
931 case NEON::BI__builtin_neon_vpadd_v:
932 case NEON::BI__builtin_neon_vpaddq_v:
933 case NEON::BI__builtin_neon_vabs_v:
934 case NEON::BI__builtin_neon_vabsq_v:
935 cgf.cgm.errorNYI(expr->getSourceRange(),
936 std::string("unimplemented AArch64 builtin call: ") +
937 ctx.BuiltinInfo.getName(ID: builtinID));
938 return mlir::Value{};
939 case NEON::BI__builtin_neon_vadd_v:
940 case NEON::BI__builtin_neon_vaddq_v: {
941 unsigned numBytes = (builtinID == NEON::BI__builtin_neon_vaddq_v) ? 16 : 8;
942 cir::VectorType byteTy =
943 cir::VectorType::getFixed(elementType: cgf.getBuilder().getUInt8Ty(), size: numBytes);
944 ops[0] = cgf.getBuilder().createBitcast(src: ops[0], newTy: byteTy);
945 ops[1] = cgf.getBuilder().createBitcast(src: ops[1], newTy: byteTy);
946 mlir::Value result = cgf.getBuilder().createXor(loc, lhs: ops[0], rhs: ops[1]);
947 return cgf.getBuilder().createBitcast(src: result, newTy: ty);
948 }
949 case NEON::BI__builtin_neon_vaddhn_v: {
950 // srcTy has double-width elements (e.g. <8 x i16> when VTy is <8 x i8>).
951 // Unsigned so createShiftRight emits lshr, not ashr.
952 cir::VectorType srcTy =
953 cgf.getBuilder().getExtendedOrTruncatedElementVectorType(
954 vt: vTy, /*isExtended=*/true, /*isSigned=*/false);
955
956 // %sum = add <4 x i32> %lhs, %rhs
957 ops[0] = cgf.getBuilder().createBitcast(src: ops[0], newTy: srcTy);
958 ops[1] = cgf.getBuilder().createBitcast(src: ops[1], newTy: srcTy);
959 mlir::Value result = cgf.getBuilder().createAdd(loc, lhs: ops[0], rhs: ops[1]);
960
961 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
962 auto wideEltTy = mlir::cast<cir::IntType>(Val: srcTy.getElementType());
963 mlir::Value shiftAmt = cgf.getBuilder().getConstantInt(
964 loc, ty: wideEltTy, value: wideEltTy.getWidth() / 2);
965 mlir::Value shiftVec =
966 emitNeonShiftVector(builder&: cgf.getBuilder(), shiftVal: shiftAmt, vecTy: srcTy, loc,
967 /*neg=*/false);
968 result = cgf.getBuilder().createShiftRight(loc, lhs: result, rhs: shiftVec);
969
970 // %res = trunc <4 x i32> %high to <4 x i16>
971 return cgf.getBuilder().createIntCast(src: result, newTy: vTy);
972 }
973 case NEON::BI__builtin_neon_vsubhn_v: {
974 // srcTy has double-width elements (e.g. <8 x i16> when VTy is <8 x i8>).
975 // Unsigned so createShiftRight emits lshr, not ashr.
976 cir::VectorType srcTy =
977 cgf.getBuilder().getExtendedOrTruncatedElementVectorType(
978 vt: vTy, /*isExtended=*/true, /*isSigned=*/false);
979
980 // %diff = sub <4 x i32> %lhs, %rhs
981 ops[0] = cgf.getBuilder().createBitcast(src: ops[0], newTy: srcTy);
982 ops[1] = cgf.getBuilder().createBitcast(src: ops[1], newTy: srcTy);
983 mlir::Value result = cgf.getBuilder().createSub(loc, lhs: ops[0], rhs: ops[1]);
984
985 // %high = lshr <4 x i32> %diff, <i32 16, i32 16, i32 16, i32 16>
986 auto wideEltTy = mlir::cast<cir::IntType>(Val: srcTy.getElementType());
987 mlir::Value shiftAmt = cgf.getBuilder().getConstantInt(
988 loc, ty: wideEltTy, value: wideEltTy.getWidth() / 2);
989 mlir::Value shiftVec =
990 emitNeonShiftVector(builder&: cgf.getBuilder(), shiftVal: shiftAmt, vecTy: srcTy, loc,
991 /*neg=*/false);
992 result = cgf.getBuilder().createShiftRight(loc, lhs: result, rhs: shiftVec);
993
994 // %res = trunc <4 x i32> %high to <4 x i16>
995 return cgf.getBuilder().createIntCast(src: result, newTy: vTy);
996 }
997 case NEON::BI__builtin_neon_vcale_v:
998 case NEON::BI__builtin_neon_vcaleq_v:
999 case NEON::BI__builtin_neon_vcalt_v:
1000 case NEON::BI__builtin_neon_vcaltq_v:
1001 case NEON::BI__builtin_neon_vcage_v:
1002 case NEON::BI__builtin_neon_vcageq_v:
1003 case NEON::BI__builtin_neon_vcagt_v:
1004 case NEON::BI__builtin_neon_vcagtq_v:
1005 cgf.cgm.errorNYI(expr->getSourceRange(),
1006 std::string("unimplemented AArch64 builtin call: ") +
1007 ctx.BuiltinInfo.getName(ID: builtinID));
1008 return mlir::Value{};
1009 case NEON::BI__builtin_neon_vceqz_v:
1010 case NEON::BI__builtin_neon_vceqzq_v:
1011 if (floating)
1012 return emitAArch64CompareBuiltinExpr(cgf, builder&: cgf.getBuilder(), loc, src: ops[0],
1013 retTy: vTy, kind: cir::FCmpOpKind::oeq);
1014 return emitAArch64CompareBuiltinExpr(cgf, builder&: cgf.getBuilder(), loc, src: ops[0],
1015 retTy: vTy, kind: cir::CmpOpKind::eq);
1016 case NEON::BI__builtin_neon_vcgez_v:
1017 case NEON::BI__builtin_neon_vcgezq_v:
1018 case NEON::BI__builtin_neon_vclez_v:
1019 case NEON::BI__builtin_neon_vclezq_v:
1020 case NEON::BI__builtin_neon_vcgtz_v:
1021 case NEON::BI__builtin_neon_vcgtzq_v:
1022 case NEON::BI__builtin_neon_vcltz_v:
1023 case NEON::BI__builtin_neon_vcltzq_v:
1024 case NEON::BI__builtin_neon_vclz_v:
1025 case NEON::BI__builtin_neon_vclzq_v:
1026 cgf.cgm.errorNYI(expr->getSourceRange(),
1027 std::string("unimplemented AArch64 builtin call: ") +
1028 ctx.BuiltinInfo.getName(ID: builtinID));
1029 return mlir::Value{};
1030 case NEON::BI__builtin_neon_vcvt_f32_v:
1031 case NEON::BI__builtin_neon_vcvtq_f32_v: {
1032 ops[0] = cgf.getBuilder().createBitcast(src: ops[0], newTy: ty);
1033 ty = getNeonType(cgf: &cgf, typeFlags: NeonTypeFlags(NeonTypeFlags::Float32, false, isQuad),
1034 hasLegalHalfType);
1035 return cgf.getBuilder().createCast(kind: cir::CastKind::int_to_float, src: ops[0], newTy: ty);
1036 }
1037 case NEON::BI__builtin_neon_vcvt_f16_s16:
1038 case NEON::BI__builtin_neon_vcvt_f16_u16:
1039 case NEON::BI__builtin_neon_vcvtq_f16_s16:
1040 case NEON::BI__builtin_neon_vcvtq_f16_u16:
1041 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
1042 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
1043 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
1044 case NEON::BI__builtin_neon_vcvtq_n_f16_u16:
1045 cgf.cgm.errorNYI(expr->getSourceRange(),
1046 std::string("unimplemented AArch64 builtin call: ") +
1047 ctx.BuiltinInfo.getName(ID: builtinID));
1048 return mlir::Value{};
1049 case NEON::BI__builtin_neon_vcvt_n_f32_v:
1050 case NEON::BI__builtin_neon_vcvt_n_f64_v:
1051 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
1052 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
1053 // The constant argument to an _n_ intrinsic always is Int32Ty.
1054 mlir::Type cstIntTy = cgf.sInt32Ty;
1055 llvm::StringRef llvmIntrName =
1056 getLLVMIntrNameNoPrefix(intrID: static_cast<llvm::Intrinsic::ID>(
1057 usgn ? llvmIntrinsic : altLLVMIntrinsic));
1058 return emitNeonCall(cgm&: cgf.getCIRGenModule(), builder&: cgf.getBuilder(),
1059 /*argTypes=*/{vTy, cstIntTy}, args&: ops, intrinsicName: llvmIntrName,
1060 /*funcResTy=*/getFloatNeonType(cgf, intTypeFlags: neonType), loc);
1061 }
1062 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
1063 case NEON::BI__builtin_neon_vcvt_n_s32_v:
1064 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
1065 case NEON::BI__builtin_neon_vcvt_n_u32_v:
1066 case NEON::BI__builtin_neon_vcvt_n_s64_v:
1067 case NEON::BI__builtin_neon_vcvt_n_u64_v:
1068 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
1069 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
1070 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
1071 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
1072 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
1073 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
1074 // The constant argument to an _n_ intrinsic always is Int32Ty.
1075 mlir::Type cstIntTy = cgf.sInt32Ty;
1076 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
1077 intrID: static_cast<llvm::Intrinsic::ID>(llvmIntrinsic));
1078 return emitNeonCall(
1079 cgm&: cgf.getCIRGenModule(), builder&: cgf.getBuilder(),
1080 /*argTypes=*/{getFloatNeonType(cgf, intTypeFlags: neonType), cstIntTy}, args&: ops,
1081 intrinsicName: llvmIntrName,
1082 /*funcResTy=*/vTy, loc);
1083 }
1084 case NEON::BI__builtin_neon_vcvt_s32_v:
1085 case NEON::BI__builtin_neon_vcvt_u32_v:
1086 case NEON::BI__builtin_neon_vcvt_s64_v:
1087 case NEON::BI__builtin_neon_vcvt_u64_v:
1088 case NEON::BI__builtin_neon_vcvt_s16_f16:
1089 case NEON::BI__builtin_neon_vcvt_u16_f16:
1090 case NEON::BI__builtin_neon_vcvtq_s32_v:
1091 case NEON::BI__builtin_neon_vcvtq_u32_v:
1092 case NEON::BI__builtin_neon_vcvtq_s64_v:
1093 case NEON::BI__builtin_neon_vcvtq_u64_v:
1094 case NEON::BI__builtin_neon_vcvtq_s16_f16:
1095 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
1096 auto ty = getFloatNeonType(cgf, intTypeFlags: neonType);
1097 // Undo the bitcast inserted by intrinsics that expand to this builtin
1098 // (e.g. vcvt_u32_f32).
1099 // TODO: While the bitcasts eventually cancel each other out, we should
1100 // avoid them altogether.
1101 ops[0] =
1102 cgf.getBuilder().createCast(loc, kind: cir::CastKind::bitcast, src: ops[0], newTy: ty);
1103 assert(!cir::MissingFeatures::emitConstrainedFPCall());
1104 // AArch64: use fptosi.sat/fptoui.sat unless under strict FP.
1105 llvm::StringRef llvmIntrName = usgn ? "fptoui.sat" : "fptosi.sat";
1106 return emitNeonCall(cgm&: cgf.getCIRGenModule(), builder&: cgf.getBuilder(),
1107 /*argTypes=*/{ty}, args&: ops, intrinsicName: llvmIntrName, funcResTy: vTy, loc);
1108 }
1109 case NEON::BI__builtin_neon_vcvta_s16_f16:
1110 case NEON::BI__builtin_neon_vcvta_s32_v:
1111 case NEON::BI__builtin_neon_vcvta_s64_v:
1112 case NEON::BI__builtin_neon_vcvta_u16_f16:
1113 case NEON::BI__builtin_neon_vcvta_u32_v:
1114 case NEON::BI__builtin_neon_vcvta_u64_v:
1115 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
1116 case NEON::BI__builtin_neon_vcvtaq_s32_v:
1117 case NEON::BI__builtin_neon_vcvtaq_s64_v:
1118 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
1119 case NEON::BI__builtin_neon_vcvtaq_u32_v:
1120 case NEON::BI__builtin_neon_vcvtaq_u64_v:
1121 case NEON::BI__builtin_neon_vcvtn_s16_f16:
1122 case NEON::BI__builtin_neon_vcvtn_s32_v:
1123 case NEON::BI__builtin_neon_vcvtn_s64_v:
1124 case NEON::BI__builtin_neon_vcvtn_u16_f16:
1125 case NEON::BI__builtin_neon_vcvtn_u32_v:
1126 case NEON::BI__builtin_neon_vcvtn_u64_v:
1127 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
1128 case NEON::BI__builtin_neon_vcvtnq_s32_v:
1129 case NEON::BI__builtin_neon_vcvtnq_s64_v:
1130 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
1131 case NEON::BI__builtin_neon_vcvtnq_u32_v:
1132 case NEON::BI__builtin_neon_vcvtnq_u64_v:
1133 case NEON::BI__builtin_neon_vcvtp_s16_f16:
1134 case NEON::BI__builtin_neon_vcvtp_s32_v:
1135 case NEON::BI__builtin_neon_vcvtp_s64_v:
1136 case NEON::BI__builtin_neon_vcvtp_u16_f16:
1137 case NEON::BI__builtin_neon_vcvtp_u32_v:
1138 case NEON::BI__builtin_neon_vcvtp_u64_v:
1139 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
1140 case NEON::BI__builtin_neon_vcvtpq_s32_v:
1141 case NEON::BI__builtin_neon_vcvtpq_s64_v:
1142 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
1143 case NEON::BI__builtin_neon_vcvtpq_u32_v:
1144 case NEON::BI__builtin_neon_vcvtpq_u64_v:
1145 case NEON::BI__builtin_neon_vcvtm_s16_f16:
1146 case NEON::BI__builtin_neon_vcvtm_s32_v:
1147 case NEON::BI__builtin_neon_vcvtm_s64_v:
1148 case NEON::BI__builtin_neon_vcvtm_u16_f16:
1149 case NEON::BI__builtin_neon_vcvtm_u32_v:
1150 case NEON::BI__builtin_neon_vcvtm_u64_v:
1151 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
1152 case NEON::BI__builtin_neon_vcvtmq_s32_v:
1153 case NEON::BI__builtin_neon_vcvtmq_s64_v:
1154 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
1155 case NEON::BI__builtin_neon_vcvtmq_u32_v:
1156 case NEON::BI__builtin_neon_vcvtmq_u64_v:
1157 cgf.cgm.errorNYI(expr->getSourceRange(),
1158 std::string("unimplemented AArch64 builtin call: ") +
1159 ctx.BuiltinInfo.getName(ID: builtinID));
1160 return mlir::Value{};
1161 case NEON::BI__builtin_neon_vcvtx_f32_v: {
1162 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
1163 intrID: static_cast<llvm::Intrinsic::ID>(llvmIntrinsic));
1164 auto v2f64 = cir::VectorType::getFixed(elementType: cgf.cgm.doubleTy, size: 2);
1165 auto v2f32 = cir::VectorType::getFixed(elementType: cgf.cgm.floatTy, size: 2);
1166 return emitNeonCall(cgm&: cgf.getCIRGenModule(), builder&: cgf.getBuilder(),
1167 /*argTypes=*/{v2f64}, args&: ops, intrinsicName: llvmIntrName,
1168 /*funcResTy=*/v2f32, loc);
1169 }
1170 case NEON::BI__builtin_neon_vext_v:
1171 case NEON::BI__builtin_neon_vextq_v:
1172 cgf.cgm.errorNYI(expr->getSourceRange(),
1173 std::string("unimplemented AArch64 builtin call: ") +
1174 ctx.BuiltinInfo.getName(ID: builtinID));
1175 return mlir::Value{};
1176 case NEON::BI__builtin_neon_vfma_v:
1177 case NEON::BI__builtin_neon_vfmaq_v: {
1178 // NEON intrinsic: vfma(q)(accumulator, multiplicand1, multiplicand2)
1179 // CIR fma: fma(multiplicand1, multiplicand2, accumulator)
1180 // Reorder arguments to match fma signature.
1181 mlir::Value op0 = cgf.getBuilder().createBitcast(src: ops[0], newTy: ty);
1182 mlir::Value op1 = cgf.getBuilder().createBitcast(src: ops[1], newTy: ty);
1183 mlir::Value op2 = cgf.getBuilder().createBitcast(src: ops[2], newTy: ty);
1184 llvm::SmallVector<mlir::Value> fmaOps = {op1, op2, op0};
1185 return emitNeonCallToOp<cir::FMAOp>(cgm&: cgf.cgm, builder&: cgf.getBuilder(), argTypes: {ty, ty, ty},
1186 args&: fmaOps, intrinsicName: std::nullopt, funcResTy: ty, loc);
1187 }
1188 case NEON::BI__builtin_neon_vld1_x2_v:
1189 case NEON::BI__builtin_neon_vld1q_x2_v:
1190 case NEON::BI__builtin_neon_vld1_x3_v:
1191 case NEON::BI__builtin_neon_vld1q_x3_v:
1192 case NEON::BI__builtin_neon_vld1_x4_v:
1193 case NEON::BI__builtin_neon_vld1q_x4_v:
1194 return emitNeonMultiVecLoad(cgf, ops, llvmIntrinsic, ty,
1195 numVecs: getNeonMultiVecCount(builtinID), loc);
1196 case NEON::BI__builtin_neon_vld1_v:
1197 case NEON::BI__builtin_neon_vld1q_v:
1198 case NEON::BI__builtin_neon_vld2_v:
1199 case NEON::BI__builtin_neon_vld2q_v:
1200 case NEON::BI__builtin_neon_vld3_v:
1201 case NEON::BI__builtin_neon_vld3q_v:
1202 case NEON::BI__builtin_neon_vld4_v:
1203 case NEON::BI__builtin_neon_vld4q_v:
1204 case NEON::BI__builtin_neon_vld2_dup_v:
1205 case NEON::BI__builtin_neon_vld2q_dup_v:
1206 case NEON::BI__builtin_neon_vld3_dup_v:
1207 case NEON::BI__builtin_neon_vld3q_dup_v:
1208 case NEON::BI__builtin_neon_vld4_dup_v:
1209 case NEON::BI__builtin_neon_vld4q_dup_v:
1210 case NEON::BI__builtin_neon_vld1_dup_v:
1211 case NEON::BI__builtin_neon_vld1q_dup_v:
1212 case NEON::BI__builtin_neon_vld2_lane_v:
1213 case NEON::BI__builtin_neon_vld2q_lane_v:
1214 case NEON::BI__builtin_neon_vld3_lane_v:
1215 case NEON::BI__builtin_neon_vld3q_lane_v:
1216 case NEON::BI__builtin_neon_vld4_lane_v:
1217 case NEON::BI__builtin_neon_vld4q_lane_v:
1218 cgf.cgm.errorNYI(expr->getSourceRange(),
1219 std::string("Reached code-path for ARM builtin call ") +
1220 ctx.BuiltinInfo.getName(ID: builtinID) +
1221 "(ARM builtins are not supported ATM)");
1222 return mlir::Value{};
1223 case NEON::BI__builtin_neon_vmovl_v: {
1224 cir::VectorType dTy =
1225 cgf.getBuilder().getExtendedOrTruncatedElementVectorType(
1226 vt: ty, /*isExtended=*/false, isSigned: !usgn);
1227 ops[0] = cgf.getBuilder().createBitcast(loc, src: ops[0], newTy: dTy);
1228 return cgf.getBuilder().createIntCast(src: ops[0], newTy: ty);
1229 }
1230 case NEON::BI__builtin_neon_vmovn_v:
1231 case NEON::BI__builtin_neon_vmull_v:
1232 case NEON::BI__builtin_neon_vpadal_v:
1233 case NEON::BI__builtin_neon_vpadalq_v:
1234 cgf.cgm.errorNYI(expr->getSourceRange(),
1235 std::string("Reached code-path for ARM builtin call ") +
1236 ctx.BuiltinInfo.getName(ID: builtinID) +
1237 "(ARM builtins are not supported ATM)");
1238 return mlir::Value{};
1239 case NEON::BI__builtin_neon_vpaddl_v:
1240 case NEON::BI__builtin_neon_vpaddlq_v: {
1241 llvm::StringRef llvmIntrName =
1242 getLLVMIntrNameNoPrefix(intrID: static_cast<llvm::Intrinsic::ID>(
1243 usgn ? llvmIntrinsic : altLLVMIntrinsic));
1244 return emitNeonCall(cgm&: cgf.getCIRGenModule(), builder&: cgf.getBuilder(),
1245 /*argTypes=*/{getNeonPairwiseWidenInputType(resType: vTy, usgn)},
1246 args&: ops, intrinsicName: llvmIntrName,
1247 /*funcResTy=*/vTy, loc);
1248 }
1249 case NEON::BI__builtin_neon_vqdmlal_v:
1250 case NEON::BI__builtin_neon_vqdmlsl_v:
1251 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
1252 case NEON::BI__builtin_neon_vqdmulh_lane_v:
1253 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
1254 case NEON::BI__builtin_neon_vqrdmulh_lane_v:
1255 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
1256 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
1257 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
1258 case NEON::BI__builtin_neon_vqrdmulh_laneq_v:
1259 cgf.cgm.errorNYI(expr->getSourceRange(),
1260 std::string("unimplemented AArch64 builtin call: ") +
1261 ctx.BuiltinInfo.getName(ID: builtinID));
1262 return mlir::Value{};
1263 case NEON::BI__builtin_neon_vqshl_n_v:
1264 case NEON::BI__builtin_neon_vqshlq_n_v: {
1265 llvm::StringRef intrName =
1266 usgn ? "aarch64.neon.uqshl" : "aarch64.neon.sqshl";
1267 return emitNeonCall(cgm&: cgf.cgm, builder&: cgf.getBuilder(), argTypes: {ty, ty}, args&: ops, intrinsicName: intrName, funcResTy: ty,
1268 loc, /*isConstrainedFPIntrinsic=*/false, /*shift=*/1,
1269 /*rightshift=*/false);
1270 }
1271 case NEON::BI__builtin_neon_vqshlu_n_v:
1272 case NEON::BI__builtin_neon_vqshluq_n_v:
1273 return emitNeonCall(cgm&: cgf.cgm, builder&: cgf.getBuilder(), argTypes: {ty, ty}, args&: ops,
1274 intrinsicName: "aarch64.neon.sqshlu", funcResTy: ty, loc,
1275 /*isConstrainedFPIntrinsic=*/false, /*shift=*/1,
1276 /*rightshift=*/false);
1277 case NEON::BI__builtin_neon_vrecpe_v:
1278 case NEON::BI__builtin_neon_vrecpeq_v:
1279 case NEON::BI__builtin_neon_vrsqrte_v:
1280 case NEON::BI__builtin_neon_vrsqrteq_v:
1281 cgf.cgm.errorNYI(expr->getSourceRange(),
1282 std::string("unimplemented AArch64 builtin call: ") +
1283 ctx.BuiltinInfo.getName(ID: builtinID));
1284 return mlir::Value{};
1285 case NEON::BI__builtin_neon_vrndi_v:
1286 case NEON::BI__builtin_neon_vrndiq_v:
1287 assert(!cir::MissingFeatures::emitConstrainedFPCall());
1288 return emitNeonCallToOp<cir::NearbyintOp>(cgm&: cgf.cgm, builder&: cgf.getBuilder(), argTypes: {ty},
1289 args&: ops, intrinsicName: std::nullopt, funcResTy: ty, loc);
1290 case NEON::BI__builtin_neon_vrshr_n_v:
1291 case NEON::BI__builtin_neon_vrshrq_n_v: {
1292 llvm::StringRef intrName =
1293 usgn ? "aarch64.neon.urshl" : "aarch64.neon.srshl";
1294 return emitNeonCall(
1295 cgm&: cgf.cgm, builder&: cgf.getBuilder(),
1296 argTypes: {ty, usgn ? getSignChangedVectorType(builder&: cgf.getBuilder(), vecTy: ty) : ty}, args&: ops,
1297 intrinsicName: intrName, funcResTy: ty, loc, /*isConstrainedFPIntrinsic=*/false,
1298 /*shift=*/1,
1299 /*rightshift=*/true);
1300 }
1301 case NEON::BI__builtin_neon_vsha512hq_u64:
1302 case NEON::BI__builtin_neon_vsha512h2q_u64:
1303 case NEON::BI__builtin_neon_vsha512su0q_u64:
1304 case NEON::BI__builtin_neon_vsha512su1q_u64:
1305 cgf.cgm.errorNYI(expr->getSourceRange(),
1306 std::string("unimplemented AArch64 builtin call: ") +
1307 ctx.BuiltinInfo.getName(ID: builtinID));
1308 return mlir::Value{};
1309 case NEON::BI__builtin_neon_vshl_n_v:
1310 case NEON::BI__builtin_neon_vshlq_n_v:
1311 return emitCommonNeonShift(builder&: cgf.getBuilder(), loc, resTy: vTy, shifTgt: ops[0], shiftAmt: ops[1],
1312 /*shiftLeft=*/true);
1313 case NEON::BI__builtin_neon_vshll_n_v: {
1314 CIRGenBuilderTy &builder = cgf.getBuilder();
1315 cir::VectorType narrowVecTy =
1316 builder.getExtendedOrTruncatedElementVectorType(vt: vTy,
1317 /*isExtended=*/false,
1318 /*isSigned=*/!usgn);
1319 mlir::Value src = builder.createBitcast(src: ops[0], newTy: narrowVecTy);
1320 mlir::Value extended = builder.createIntCast(src, newTy: vTy);
1321 return emitCommonNeonShift(builder, loc, resTy: vTy, shifTgt: extended, shiftAmt: ops[1],
1322 /*shiftLeft=*/true);
1323 }
1324 case NEON::BI__builtin_neon_vshrn_n_v: {
1325 CIRGenBuilderTy &builder = cgf.getBuilder();
1326 cir::VectorType wideVecTy =
1327 builder.getExtendedOrTruncatedElementVectorType(vt: vTy,
1328 /*isExtended=*/true,
1329 /*isSigned=*/!usgn);
1330 mlir::Value src = builder.createBitcast(src: ops[0], newTy: wideVecTy);
1331 mlir::Value shifted = emitCommonNeonShift(builder, loc, resTy: wideVecTy, shifTgt: src,
1332 shiftAmt: ops[1], /*shiftLeft=*/false);
1333 return builder.createIntCast(src: shifted, newTy: vTy);
1334 }
1335 case NEON::BI__builtin_neon_vshr_n_v:
1336 case NEON::BI__builtin_neon_vshrq_n_v:
1337 return emitNeonRShiftImm(cgf, shiftVec: ops[0], shiftVal: ops[1], vecTy: vTy, usgn: isUnsigned, loc);
1338 case NEON::BI__builtin_neon_vst1_v:
1339 case NEON::BI__builtin_neon_vst1q_v:
1340 case NEON::BI__builtin_neon_vst2_v:
1341 case NEON::BI__builtin_neon_vst2q_v:
1342 case NEON::BI__builtin_neon_vst3_v:
1343 case NEON::BI__builtin_neon_vst3q_v:
1344 case NEON::BI__builtin_neon_vst4_v:
1345 case NEON::BI__builtin_neon_vst4q_v:
1346 case NEON::BI__builtin_neon_vst2_lane_v:
1347 case NEON::BI__builtin_neon_vst2q_lane_v:
1348 case NEON::BI__builtin_neon_vst3_lane_v:
1349 case NEON::BI__builtin_neon_vst3q_lane_v:
1350 case NEON::BI__builtin_neon_vst4_lane_v:
1351 case NEON::BI__builtin_neon_vst4q_lane_v:
1352 case NEON::BI__builtin_neon_vsm3partw1q_u32:
1353 case NEON::BI__builtin_neon_vsm3partw2q_u32:
1354 case NEON::BI__builtin_neon_vsm3ss1q_u32:
1355 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
1356 case NEON::BI__builtin_neon_vsm4eq_u32:
1357 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
1358 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
1359 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
1360 case NEON::BI__builtin_neon_vsm3tt2bq_u32:
1361 cgf.cgm.errorNYI(expr->getSourceRange(),
1362 std::string("unimplemented AArch64 builtin call: ") +
1363 ctx.BuiltinInfo.getName(ID: builtinID));
1364 return mlir::Value{};
1365 case NEON::BI__builtin_neon_vst1_x2_v:
1366 case NEON::BI__builtin_neon_vst1q_x2_v:
1367 case NEON::BI__builtin_neon_vst1_x3_v:
1368 case NEON::BI__builtin_neon_vst1q_x3_v:
1369 case NEON::BI__builtin_neon_vst1_x4_v:
1370 case NEON::BI__builtin_neon_vst1q_x4_v: {
1371 // The builtin call has the pointer first, but the AArch64 st1x2/3/4
1372 // intrinsics take the vector operands first and the pointer last.
1373 std::rotate(first: ops.begin(), middle: ops.begin() + 1, last: ops.end());
1374 llvm::SmallVector<mlir::Type> argTypes(ops.size() - 1, ty);
1375 argTypes.push_back(Elt: cgf.getBuilder().getVoidPtrTy());
1376 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
1377 intrID: static_cast<llvm::Intrinsic::ID>(llvmIntrinsic));
1378 return emitNeonCall(cgm&: cgf.getCIRGenModule(), builder&: cgf.getBuilder(), argTypes, args&: ops,
1379 intrinsicName: llvmIntrName, funcResTy: cgf.cgm.voidTy, loc);
1380 }
1381 case NEON::BI__builtin_neon_vtrn_v:
1382 case NEON::BI__builtin_neon_vtrnq_v:
1383 case NEON::BI__builtin_neon_vtst_v:
1384 case NEON::BI__builtin_neon_vtstq_v:
1385 case NEON::BI__builtin_neon_vuzp_v:
1386 case NEON::BI__builtin_neon_vuzpq_v:
1387 case NEON::BI__builtin_neon_vxarq_u64:
1388 case NEON::BI__builtin_neon_vzip_v:
1389 case NEON::BI__builtin_neon_vzipq_v:
1390 case NEON::BI__builtin_neon_vdot_s32:
1391 case NEON::BI__builtin_neon_vdot_u32:
1392 case NEON::BI__builtin_neon_vdotq_s32:
1393 case NEON::BI__builtin_neon_vdotq_u32:
1394 case NEON::BI__builtin_neon_vfmlal_low_f16:
1395 case NEON::BI__builtin_neon_vfmlalq_low_f16:
1396 case NEON::BI__builtin_neon_vfmlsl_low_f16:
1397 case NEON::BI__builtin_neon_vfmlslq_low_f16:
1398 case NEON::BI__builtin_neon_vfmlal_high_f16:
1399 case NEON::BI__builtin_neon_vfmlalq_high_f16:
1400 case NEON::BI__builtin_neon_vfmlsl_high_f16:
1401 case NEON::BI__builtin_neon_vfmlslq_high_f16:
1402 case NEON::BI__builtin_neon_vmmlaq_s32:
1403 case NEON::BI__builtin_neon_vmmlaq_u32:
1404 cgf.cgm.errorNYI(expr->getSourceRange(),
1405 std::string("unimplemented AArch64 builtin call: ") +
1406 ctx.BuiltinInfo.getName(ID: builtinID));
1407 return mlir::Value{};
1408 case NEON::BI__builtin_neon_vmul_v:
1409 case NEON::BI__builtin_neon_vmulq_v:
1410 return cgf.getBuilder().emitIntrinsicCallOp(loc, str: "aarch64.neon.pmul", resTy: vTy,
1411 op&: ops);
1412 case NEON::BI__builtin_neon_vusmmlaq_s32:
1413 case NEON::BI__builtin_neon_vusdot_s32:
1414 case NEON::BI__builtin_neon_vusdotq_s32:
1415 case NEON::BI__builtin_neon_vbfdot_f32:
1416 case NEON::BI__builtin_neon_vbfdotq_f32:
1417 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32:
1418 cgf.cgm.errorNYI(expr->getSourceRange(),
1419 std::string("unimplemented AArch64 builtin call: ") +
1420 ctx.BuiltinInfo.getName(ID: builtinID));
1421 return mlir::Value{};
1422 }
1423
1424 // The switch stmt is intended to help catch NYI cases and will be removed
1425 // once the CIR implementation is complete. Avoid adding specialized
1426 // code in cases - that should only be required for a handful of examples.
1427 switch (builtinID) {
1428 default:
1429 cgf.cgm.errorNYI(expr->getSourceRange(),
1430 std::string("unimplemented AArch64 builtin call: ") +
1431 cgf.getContext().BuiltinInfo.getName(ID: builtinID));
1432 break;
1433 case NEON::BI__builtin_neon_vrnd32x_f32:
1434 case NEON::BI__builtin_neon_vrnd32xq_f32:
1435 case NEON::BI__builtin_neon_vrnd32x_f64:
1436 case NEON::BI__builtin_neon_vrnd32xq_f64:
1437 case NEON::BI__builtin_neon_vrnd32z_f32:
1438 case NEON::BI__builtin_neon_vrnd32zq_f32:
1439 case NEON::BI__builtin_neon_vrnd32z_f64:
1440 case NEON::BI__builtin_neon_vrnd32zq_f64:
1441 case NEON::BI__builtin_neon_vrnd64x_f32:
1442 case NEON::BI__builtin_neon_vrnd64xq_f32:
1443 case NEON::BI__builtin_neon_vrnd64x_f64:
1444 case NEON::BI__builtin_neon_vrnd64xq_f64:
1445 case NEON::BI__builtin_neon_vrnd64z_f32:
1446 case NEON::BI__builtin_neon_vrnd64zq_f32:
1447 case NEON::BI__builtin_neon_vrnd64z_f64:
1448 case NEON::BI__builtin_neon_vrnd64zq_f64: {
1449 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
1450 intrID: static_cast<llvm::Intrinsic::ID>(llvmIntrinsic));
1451 mlir::Value result =
1452 emitNeonCall(cgm&: cgf.cgm, builder&: cgf.getBuilder(), /*argTypes=*/{vTy}, args&: ops,
1453 intrinsicName: llvmIntrName, /*funcResTy=*/vTy, loc);
1454 mlir::Type resultType = cgf.convertType(t: expr->getType());
1455 return cgf.getBuilder().createBitcast(src: result, newTy: resultType);
1456 }
1457 case NEON::BI__builtin_neon_vhadd_v:
1458 case NEON::BI__builtin_neon_vhaddq_v:
1459 case NEON::BI__builtin_neon_vhsub_v:
1460 case NEON::BI__builtin_neon_vhsubq_v:
1461 case NEON::BI__builtin_neon_vqadd_v:
1462 case NEON::BI__builtin_neon_vqaddq_v:
1463 case NEON::BI__builtin_neon_vrhadd_v:
1464 case NEON::BI__builtin_neon_vrhaddq_v:
1465 case NEON::BI__builtin_neon_vqsub_v:
1466 case NEON::BI__builtin_neon_vqsubq_v:
1467 case NEON::BI__builtin_neon_vshl_v:
1468 case NEON::BI__builtin_neon_vshlq_v:
1469 case NEON::BI__builtin_neon_vraddhn_v:
1470 case NEON::BI__builtin_neon_vrsubhn_v:
1471 case NEON::BI__builtin_neon_vqshl_v:
1472 case NEON::BI__builtin_neon_vqshlq_v: {
1473 // Pick the signed/unsigned intrinsic when the builtin has both
1474 // (UnsignedAlts); otherwise there is a single intrinsic.
1475 unsigned intrinsic =
1476 ((modifier & UnsignedAlts) && !usgn) ? altLLVMIntrinsic : llvmIntrinsic;
1477 llvm::StringRef llvmIntrName =
1478 getLLVMIntrNameNoPrefix(intrID: static_cast<llvm::Intrinsic::ID>(intrinsic));
1479
1480 cir::VectorType argTy =
1481 deriveNeonBinaryArgType(builder&: cgf.getBuilder(), modifier, vTy);
1482
1483 mlir::Value result =
1484 emitNeonCall(cgm&: cgf.getCIRGenModule(), builder&: cgf.getBuilder(),
1485 /*argTypes=*/{argTy, argTy}, args&: ops, intrinsicName: llvmIntrName,
1486 /*funcResTy=*/vTy, loc);
1487 mlir::Type resultType = cgf.convertType(t: expr->getType());
1488 return cgf.getBuilder().createBitcast(src: result, newTy: resultType);
1489 }
1490 }
1491
1492 // NYI
1493 return nullptr;
1494}
1495
1496bool CIRGenFunction::getAArch64SVEProcessedOperands(
1497 unsigned builtinID, const CallExpr *expr, SmallVectorImpl<mlir::Value> &ops,
1498 SVETypeFlags typeFlags) {
1499 // Find out if any arguments are required to be integer constant expressions.
1500 unsigned iceArguments = 0;
1501 ASTContext::GetBuiltinTypeError error;
1502 getContext().GetBuiltinType(ID: builtinID, Error&: error, IntegerConstantArgs: &iceArguments);
1503 assert(error == ASTContext::GE_None && "Should not codegen an error");
1504
1505 for (unsigned i = 0, e = expr->getNumArgs(); i != e; i++) {
1506 bool isIce = iceArguments & (1 << i);
1507 mlir::Value arg = emitScalarExpr(e: expr->getArg(Arg: i));
1508
1509 if (isIce) {
1510 cgm.errorNYI(expr->getSourceRange(),
1511 std::string("unimplemented AArch64 builtin call: ") +
1512 getContext().BuiltinInfo.getName(ID: builtinID));
1513 }
1514
1515 // FIXME: Handle types like svint16x2_t, which are currently incorrectly
1516 // converted to i32. These should be treated as structs and unpacked.
1517
1518 ops.push_back(Elt: arg);
1519 }
1520 return true;
1521}
1522
1523// Reinterpret the input predicate so that it can be used to correctly isolate
1524// the elements of the specified datatype.
1525mlir::Value CIRGenFunction::emitSVEPredicateCast(mlir::Value pred,
1526 unsigned minNumElts,
1527 mlir::Location loc) {
1528
1529 // TODO: Handle "aarch64.svcount" once we get round to supporting SME.
1530
1531 auto retTy = cir::VectorType::getScalable(elementType: builder.getUIntNTy(n: 1), size: minNumElts);
1532 if (pred.getType() == retTy)
1533 return pred;
1534
1535 llvm::Intrinsic::ID intID;
1536 switch (minNumElts) {
1537 default:
1538 llvm_unreachable("unsupported element count!");
1539 case 1:
1540 case 2:
1541 case 4:
1542 case 8:
1543 intID = Intrinsic::aarch64_sve_convert_from_svbool;
1544 break;
1545 case 16:
1546 intID = Intrinsic::aarch64_sve_convert_to_svbool;
1547 break;
1548 }
1549
1550 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(intrID: intID);
1551 auto call = builder.emitIntrinsicCallOp(loc, str: llvmIntrName, resTy: retTy,
1552 op: mlir::ValueRange{pred});
1553 assert(call.getType() == retTy && "Unexpected return type!");
1554 return call;
1555}
1556
1557//===----------------------------------------------------------------------===//
1558// SVE helpers
1559//===----------------------------------------------------------------------===//
1560// Get the minimum number of elements in an SVE vector for the given element
1561// type. The actual number of elements in the vector would be an integer (power
1562// of two) multiple of this value.
1563static unsigned getSVEMinEltCount(clang::SVETypeFlags::EltType sveType) {
1564 switch (sveType) {
1565 default:
1566 llvm_unreachable("Invalid SVETypeFlag!");
1567
1568 case SVETypeFlags::EltTyInt8:
1569 return 16;
1570 case SVETypeFlags::EltTyInt16:
1571 return 8;
1572 case SVETypeFlags::EltTyInt32:
1573 return 4;
1574 case SVETypeFlags::EltTyInt64:
1575 return 2;
1576
1577 case SVETypeFlags::EltTyMFloat8:
1578 return 16;
1579 case SVETypeFlags::EltTyFloat16:
1580 case SVETypeFlags::EltTyBFloat16:
1581 return 8;
1582 case SVETypeFlags::EltTyFloat32:
1583 return 4;
1584 case SVETypeFlags::EltTyFloat64:
1585 return 2;
1586
1587 case SVETypeFlags::EltTyBool8:
1588 return 16;
1589 case SVETypeFlags::EltTyBool16:
1590 return 8;
1591 case SVETypeFlags::EltTyBool32:
1592 return 4;
1593 case SVETypeFlags::EltTyBool64:
1594 return 2;
1595 }
1596}
1597
1598cir::VectorType CIRGenFunction::getSVEType(const SVETypeFlags &typeFlags) {
1599 switch (typeFlags.getEltType()) {
1600 default:
1601 llvm_unreachable("Invalid SVETypeFlag!");
1602
1603 case SVETypeFlags::EltTyInt8:
1604 return cir::VectorType::getScalable(elementType: builder.getUInt8Ty(), size: 16);
1605 case SVETypeFlags::EltTyInt16:
1606 return cir::VectorType::getScalable(elementType: builder.getUInt16Ty(), size: 8);
1607 case SVETypeFlags::EltTyInt32:
1608 return cir::VectorType::getScalable(elementType: builder.getUInt32Ty(), size: 4);
1609 case SVETypeFlags::EltTyInt64:
1610 return cir::VectorType::getScalable(elementType: builder.getUInt64Ty(), size: 2);
1611
1612 case SVETypeFlags::EltTyMFloat8:
1613 return cir::VectorType::getScalable(elementType: builder.getUInt8Ty(), size: 16);
1614 case SVETypeFlags::EltTyFloat16:
1615 return cir::VectorType::getScalable(elementType: builder.getFp16Ty(), size: 8);
1616 case SVETypeFlags::EltTyBFloat16:
1617 return cir::VectorType::getScalable(elementType: builder.getBfloat16Ty(), size: 8);
1618 case SVETypeFlags::EltTyFloat32:
1619 return cir::VectorType::getScalable(elementType: builder.getSingleTy(), size: 4);
1620 case SVETypeFlags::EltTyFloat64:
1621 return cir::VectorType::getScalable(elementType: builder.getDoubleTy(), size: 2);
1622
1623 case SVETypeFlags::EltTyBool8:
1624 return cir::VectorType::getScalable(elementType: builder.getUIntNTy(n: 1), size: 16);
1625 case SVETypeFlags::EltTyBool16:
1626 return cir::VectorType::getScalable(elementType: builder.getUIntNTy(n: 1), size: 8);
1627 case SVETypeFlags::EltTyBool32:
1628 return cir::VectorType::getScalable(elementType: builder.getUIntNTy(n: 1), size: 4);
1629 case SVETypeFlags::EltTyBool64:
1630 return cir::VectorType::getScalable(elementType: builder.getUIntNTy(n: 1), size: 2);
1631 }
1632}
1633
1634// TODO(cir): Share with OGCG
1635constexpr unsigned sveBitsPerBlock = 128;
1636
1637static cir::VectorType getSVEVectorForElementType(CIRGenModule &cgm,
1638 mlir::Type eltTy) {
1639 unsigned numElts =
1640 sveBitsPerBlock / cgm.getDataLayout().getTypeSizeInBits(ty: eltTy);
1641 return cir::VectorType::getScalable(elementType: eltTy, size: numElts);
1642}
1643
1644//===----------------------------------------------------------------------===//
1645// SVE helpers
1646//===----------------------------------------------------------------------===//
1647std::optional<mlir::Value>
1648CIRGenFunction::emitAArch64SVEBuiltinExpr(unsigned builtinID,
1649 const CallExpr *expr) {
1650 mlir::Type ty = convertType(t: expr->getType());
1651
1652 if (builtinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
1653 builtinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
1654 cgm.errorNYI(expr->getSourceRange(),
1655 std::string("unimplemented AArch64 builtin call: ") +
1656 getContext().BuiltinInfo.getName(ID: builtinID));
1657 return mlir::Value{};
1658 }
1659
1660 assert(!cir::MissingFeatures::aarch64SVEIntrinsics());
1661
1662 auto *builtinIntrInfo =
1663 findARMVectorIntrinsicInMap(intrinsicMap: ArrayRef(aarch64SVEIntrinsicMap), builtinID,
1664 mapProvenSorted&: aarch64SVEIntrinsicsProvenSorted);
1665
1666 // The operands of the builtin call
1667 llvm::SmallVector<mlir::Value> ops;
1668
1669 SVETypeFlags typeFlags(builtinIntrInfo->TypeModifier);
1670 if (!CIRGenFunction::getAArch64SVEProcessedOperands(builtinID, expr, ops,
1671 typeFlags))
1672 return mlir::Value{};
1673
1674 if (typeFlags.isLoad() || typeFlags.isStore() || typeFlags.isGatherLoad() ||
1675 typeFlags.isScatterStore() || typeFlags.isPrefetch() ||
1676 typeFlags.isGatherPrefetch() || typeFlags.isStructLoad() ||
1677 typeFlags.isStructStore() || typeFlags.isTupleSet() ||
1678 typeFlags.isTupleGet() || typeFlags.isTupleCreate() ||
1679 typeFlags.isUndef())
1680 cgm.errorNYI(expr->getSourceRange(),
1681 std::string("unimplemented AArch64 builtin call: ") +
1682 getContext().BuiltinInfo.getName(ID: builtinID));
1683
1684 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
1685
1686 // Handle built-ins for which there is a corresponding LLVM Intrinsic.
1687 // -------------------------------------------------------------------
1688 if (builtinIntrInfo->LLVMIntrinsic != 0) {
1689 // Emit set FPMR for intrinsics that require it.
1690 if (typeFlags.setsFPMR())
1691 cgm.errorNYI(expr->getSourceRange(),
1692 std::string("unimplemented AArch64 builtin call: ") +
1693 getContext().BuiltinInfo.getName(ID: builtinID));
1694
1695 // Zero-ing predication
1696 if (typeFlags.getMergeType() == SVETypeFlags::MergeZeroExp) {
1697 auto null = builder.getNullValue(ty: convertType(t: expr->getType()),
1698 loc: getLoc(srcLoc: expr->getExprLoc()));
1699 ops.insert(I: ops.begin(), Elt: null);
1700 }
1701
1702 if (typeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
1703 ops.insert(I: ops.begin(),
1704 Elt: builder.getConstant(loc, attr: cir::UndefAttr::get(type: ty)));
1705
1706 // Some ACLE builtins leave out the argument to specify the predicate
1707 // pattern, which is expected to be expanded to an SV_ALL pattern.
1708 if (typeFlags.isAppendSVALL())
1709 cgm.errorNYI(expr->getSourceRange(),
1710 std::string("unimplemented AArch64 builtin call: ") +
1711 getContext().BuiltinInfo.getName(ID: builtinID));
1712 if (typeFlags.isInsertOp1SVALL())
1713 cgm.errorNYI(expr->getSourceRange(),
1714 std::string("unimplemented AArch64 builtin call: ") +
1715 getContext().BuiltinInfo.getName(ID: builtinID));
1716
1717 // Predicates must match the main datatype.
1718 for (mlir::Value &op : ops)
1719 if (auto predTy = dyn_cast<cir::VectorType>(Val: op.getType()))
1720 if (auto cirInt = dyn_cast<cir::IntType>(Val: predTy.getElementType()))
1721 if (cirInt.getWidth() == 1)
1722 op = emitSVEPredicateCast(
1723 pred: op, minNumElts: getSVEMinEltCount(sveType: typeFlags.getEltType()), loc);
1724
1725 // Splat scalar operand to vector (intrinsics with _n infix)
1726 if (typeFlags.hasSplatOperand()) {
1727 unsigned opNo = typeFlags.getSplatOperand();
1728 ops[opNo] = cir::VecSplatOp::create(
1729 builder, location: loc, result: getSVEVectorForElementType(cgm, eltTy: ops[opNo].getType()),
1730 value: ops[opNo]);
1731 }
1732
1733 if (typeFlags.isReverseCompare())
1734 cgm.errorNYI(expr->getSourceRange(),
1735 std::string("unimplemented AArch64 builtin call: ") +
1736 getContext().BuiltinInfo.getName(ID: builtinID));
1737 if (typeFlags.isReverseUSDOT())
1738 cgm.errorNYI(expr->getSourceRange(),
1739 std::string("unimplemented AArch64 builtin call: ") +
1740 getContext().BuiltinInfo.getName(ID: builtinID));
1741 if (typeFlags.isReverseMergeAnyBinOp() &&
1742 typeFlags.getMergeType() == SVETypeFlags::MergeAny)
1743 cgm.errorNYI(expr->getSourceRange(),
1744 std::string("unimplemented AArch64 builtin call: ") +
1745 getContext().BuiltinInfo.getName(ID: builtinID));
1746 if (typeFlags.isReverseMergeAnyAccOp() &&
1747 typeFlags.getMergeType() == SVETypeFlags::MergeAny)
1748 cgm.errorNYI(expr->getSourceRange(),
1749 std::string("unimplemented AArch64 builtin call: ") +
1750 getContext().BuiltinInfo.getName(ID: builtinID));
1751
1752 // Predicated intrinsics with _z suffix.
1753 if (typeFlags.getMergeType() == SVETypeFlags::MergeZero) {
1754 cgm.errorNYI(expr->getSourceRange(),
1755 std::string("unimplemented AArch64 builtin call: ") +
1756 getContext().BuiltinInfo.getName(ID: builtinID));
1757 }
1758
1759 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
1760 intrID: static_cast<llvm::Intrinsic::ID>(builtinIntrInfo->LLVMIntrinsic));
1761 auto retTy = convertType(t: expr->getType());
1762
1763 auto call = builder.emitIntrinsicCallOp(loc, str: llvmIntrName, resTy: retTy,
1764 op: mlir::ValueRange{ops});
1765 if (call.getType() == retTy)
1766 return call;
1767
1768 // Predicate results must be converted to svbool_t.
1769 if (isa<mlir::VectorType>(Val: retTy) &&
1770 cast<mlir::VectorType>(Val&: retTy).isScalable())
1771 cgm.errorNYI(expr->getSourceRange(),
1772 std::string("unimplemented AArch64 builtin call: ") +
1773 getContext().BuiltinInfo.getName(ID: builtinID));
1774 // TODO Handle struct types, e.g. svint8x2_t (update the converter first).
1775
1776 llvm_unreachable("unsupported element count!");
1777 }
1778
1779 // Handle the remaining built-ins.
1780 // -------------------------------
1781 switch (builtinID) {
1782 default:
1783 return std::nullopt;
1784
1785 case SVE::BI__builtin_sve_svreinterpret_b:
1786 case SVE::BI__builtin_sve_svreinterpret_c:
1787 case SVE::BI__builtin_sve_svpsel_lane_b8:
1788 case SVE::BI__builtin_sve_svpsel_lane_b16:
1789 case SVE::BI__builtin_sve_svpsel_lane_b32:
1790 case SVE::BI__builtin_sve_svpsel_lane_b64:
1791 case SVE::BI__builtin_sve_svpsel_lane_c8:
1792 case SVE::BI__builtin_sve_svpsel_lane_c16:
1793 case SVE::BI__builtin_sve_svpsel_lane_c32:
1794 case SVE::BI__builtin_sve_svpsel_lane_c64:
1795 case SVE::BI__builtin_sve_svmov_b_z:
1796 case SVE::BI__builtin_sve_svnot_b_z:
1797 case SVE::BI__builtin_sve_svmovlb_u16:
1798 case SVE::BI__builtin_sve_svmovlb_u32:
1799 case SVE::BI__builtin_sve_svmovlb_u64:
1800 case SVE::BI__builtin_sve_svmovlb_s16:
1801 case SVE::BI__builtin_sve_svmovlb_s32:
1802 case SVE::BI__builtin_sve_svmovlb_s64:
1803 case SVE::BI__builtin_sve_svmovlt_u16:
1804 case SVE::BI__builtin_sve_svmovlt_u32:
1805 case SVE::BI__builtin_sve_svmovlt_u64:
1806 case SVE::BI__builtin_sve_svmovlt_s16:
1807 case SVE::BI__builtin_sve_svmovlt_s32:
1808 case SVE::BI__builtin_sve_svmovlt_s64:
1809 case SVE::BI__builtin_sve_svpmullt_u16:
1810 case SVE::BI__builtin_sve_svpmullt_u64:
1811 case SVE::BI__builtin_sve_svpmullt_n_u16:
1812 case SVE::BI__builtin_sve_svpmullt_n_u64:
1813 case SVE::BI__builtin_sve_svpmullb_u16:
1814 case SVE::BI__builtin_sve_svpmullb_u64:
1815 case SVE::BI__builtin_sve_svpmullb_n_u16:
1816 case SVE::BI__builtin_sve_svpmullb_n_u64:
1817 cgm.errorNYI(expr->getSourceRange(),
1818 std::string("unimplemented AArch64 builtin call: ") +
1819 getContext().BuiltinInfo.getName(ID: builtinID));
1820 return mlir::Value{};
1821
1822 case SVE::BI__builtin_sve_svdup_n_b8:
1823 case SVE::BI__builtin_sve_svdup_n_b16:
1824 case SVE::BI__builtin_sve_svdup_n_b32:
1825 case SVE::BI__builtin_sve_svdup_n_b64: {
1826 // Cast from cir.bool (input type) to cir.int<u, 1> (element type of the
1827 // result vector).
1828 auto dup = builder.createBitcast(src: ops[0], newTy: builder.getUIntNTy(n: 1));
1829
1830 // Splat
1831 dup = cir::VecSplatOp::create(builder, location: loc, result: getSVEType(typeFlags), value: dup);
1832
1833 // Cast to svbool_t, i.e. <vscale x 16 x i1>. The actual result could be
1834 // e.g. <vscale x 8 x i1> (for b16), but only svbool_t (i.e. full
1835 // predicate register) is "storable" (as per SVE ABI).
1836 return builtinID == SVE::BI__builtin_sve_svdup_n_b8
1837 ? dup
1838 : emitSVEPredicateCast(pred: dup, minNumElts: 16, loc);
1839 }
1840
1841 case SVE::BI__builtin_sve_svdupq_n_b8:
1842 case SVE::BI__builtin_sve_svdupq_n_b16:
1843 case SVE::BI__builtin_sve_svdupq_n_b32:
1844 case SVE::BI__builtin_sve_svdupq_n_b64:
1845 case SVE::BI__builtin_sve_svdupq_n_u8:
1846 case SVE::BI__builtin_sve_svdupq_n_s8:
1847 case SVE::BI__builtin_sve_svdupq_n_u64:
1848 case SVE::BI__builtin_sve_svdupq_n_f64:
1849 case SVE::BI__builtin_sve_svdupq_n_s64:
1850 case SVE::BI__builtin_sve_svdupq_n_u16:
1851 case SVE::BI__builtin_sve_svdupq_n_f16:
1852 case SVE::BI__builtin_sve_svdupq_n_bf16:
1853 case SVE::BI__builtin_sve_svdupq_n_s16:
1854 case SVE::BI__builtin_sve_svdupq_n_u32:
1855 case SVE::BI__builtin_sve_svdupq_n_f32:
1856 case SVE::BI__builtin_sve_svdupq_n_s32:
1857 cgm.errorNYI(expr->getSourceRange(),
1858 std::string("unimplemented AArch64 builtin call: ") +
1859 getContext().BuiltinInfo.getName(ID: builtinID));
1860 return mlir::Value{};
1861
1862 case SVE::BI__builtin_sve_svpfalse_b:
1863 return builder.getNullValue(ty: convertType(t: expr->getType()),
1864 loc: getLoc(srcLoc: expr->getExprLoc()));
1865
1866 case SVE::BI__builtin_sve_svpfalse_c:
1867 cgm.errorNYI(expr->getSourceRange(),
1868 std::string("unimplemented AArch64 builtin call: ") +
1869 getContext().BuiltinInfo.getName(ID: builtinID));
1870 return mlir::Value{};
1871
1872 case SVE::BI__builtin_sve_svlen_u8:
1873 case SVE::BI__builtin_sve_svlen_s8:
1874 return genVscaleTimesFactor(loc, builder, cirTy: convertType(t: expr->getType()), scalingFactor: 16);
1875
1876 case SVE::BI__builtin_sve_svlen_u16:
1877 case SVE::BI__builtin_sve_svlen_s16:
1878 case SVE::BI__builtin_sve_svlen_f16:
1879 case SVE::BI__builtin_sve_svlen_bf16:
1880 return genVscaleTimesFactor(loc, builder, cirTy: convertType(t: expr->getType()), scalingFactor: 8);
1881
1882 case SVE::BI__builtin_sve_svlen_u32:
1883 case SVE::BI__builtin_sve_svlen_s32:
1884 case SVE::BI__builtin_sve_svlen_f32:
1885 return genVscaleTimesFactor(loc, builder, cirTy: convertType(t: expr->getType()), scalingFactor: 4);
1886
1887 case SVE::BI__builtin_sve_svlen_u64:
1888 case SVE::BI__builtin_sve_svlen_s64:
1889 case SVE::BI__builtin_sve_svlen_f64:
1890 return genVscaleTimesFactor(loc, builder, cirTy: convertType(t: expr->getType()), scalingFactor: 2);
1891
1892 case SVE::BI__builtin_sve_svtbl2_u8:
1893 case SVE::BI__builtin_sve_svtbl2_s8:
1894 case SVE::BI__builtin_sve_svtbl2_u16:
1895 case SVE::BI__builtin_sve_svtbl2_s16:
1896 case SVE::BI__builtin_sve_svtbl2_u32:
1897 case SVE::BI__builtin_sve_svtbl2_s32:
1898 case SVE::BI__builtin_sve_svtbl2_u64:
1899 case SVE::BI__builtin_sve_svtbl2_s64:
1900 case SVE::BI__builtin_sve_svtbl2_f16:
1901 case SVE::BI__builtin_sve_svtbl2_bf16:
1902 case SVE::BI__builtin_sve_svtbl2_f32:
1903 case SVE::BI__builtin_sve_svtbl2_f64:
1904 case SVE::BI__builtin_sve_svset_neonq_s8:
1905 case SVE::BI__builtin_sve_svset_neonq_s16:
1906 case SVE::BI__builtin_sve_svset_neonq_s32:
1907 case SVE::BI__builtin_sve_svset_neonq_s64:
1908 case SVE::BI__builtin_sve_svset_neonq_u8:
1909 case SVE::BI__builtin_sve_svset_neonq_u16:
1910 case SVE::BI__builtin_sve_svset_neonq_u32:
1911 case SVE::BI__builtin_sve_svset_neonq_u64:
1912 case SVE::BI__builtin_sve_svset_neonq_f16:
1913 case SVE::BI__builtin_sve_svset_neonq_f32:
1914 case SVE::BI__builtin_sve_svset_neonq_f64:
1915 case SVE::BI__builtin_sve_svset_neonq_bf16:
1916 case SVE::BI__builtin_sve_svset_neonq_mf8:
1917 case SVE::BI__builtin_sve_svget_neonq_s8:
1918 case SVE::BI__builtin_sve_svget_neonq_s16:
1919 case SVE::BI__builtin_sve_svget_neonq_s32:
1920 case SVE::BI__builtin_sve_svget_neonq_s64:
1921 case SVE::BI__builtin_sve_svget_neonq_u8:
1922 case SVE::BI__builtin_sve_svget_neonq_u16:
1923 case SVE::BI__builtin_sve_svget_neonq_u32:
1924 case SVE::BI__builtin_sve_svget_neonq_u64:
1925 case SVE::BI__builtin_sve_svget_neonq_f16:
1926 case SVE::BI__builtin_sve_svget_neonq_f32:
1927 case SVE::BI__builtin_sve_svget_neonq_f64:
1928 case SVE::BI__builtin_sve_svget_neonq_bf16:
1929 case SVE::BI__builtin_sve_svget_neonq_mf8:
1930 case SVE::BI__builtin_sve_svdup_neonq_s8:
1931 case SVE::BI__builtin_sve_svdup_neonq_s16:
1932 case SVE::BI__builtin_sve_svdup_neonq_s32:
1933 case SVE::BI__builtin_sve_svdup_neonq_s64:
1934 case SVE::BI__builtin_sve_svdup_neonq_u8:
1935 case SVE::BI__builtin_sve_svdup_neonq_u16:
1936 case SVE::BI__builtin_sve_svdup_neonq_u32:
1937 case SVE::BI__builtin_sve_svdup_neonq_u64:
1938 case SVE::BI__builtin_sve_svdup_neonq_f16:
1939 case SVE::BI__builtin_sve_svdup_neonq_f32:
1940 case SVE::BI__builtin_sve_svdup_neonq_f64:
1941 case SVE::BI__builtin_sve_svdup_neonq_bf16:
1942 case SVE::BI__builtin_sve_svdup_neonq_mf8:
1943 cgm.errorNYI(expr->getSourceRange(),
1944 std::string("unimplemented AArch64 builtin call: ") +
1945 getContext().BuiltinInfo.getName(ID: builtinID));
1946 return mlir::Value{};
1947 }
1948
1949 // Unreachable: All cases in the switch above return.
1950}
1951
1952std::optional<mlir::Value>
1953CIRGenFunction::emitAArch64SMEBuiltinExpr(unsigned builtinID,
1954 const CallExpr *expr) {
1955 assert(!cir::MissingFeatures::aarch64SMEIntrinsics());
1956
1957 cgm.errorNYI(expr->getSourceRange(),
1958 std::string("unimplemented AArch64 builtin call: ") +
1959 getContext().BuiltinInfo.getName(ID: builtinID));
1960 return mlir::Value{};
1961}
1962
1963// Some intrinsics are equivalent for codegen.
1964static const std::pair<unsigned, unsigned> neonEquivalentIntrinsicMap[] = {
1965 {
1966 NEON::BI__builtin_neon_vabd_f16,
1967 NEON::BI__builtin_neon_vabd_v,
1968 },
1969 {
1970 NEON::BI__builtin_neon_vabdq_f16,
1971 NEON::BI__builtin_neon_vabdq_v,
1972 },
1973 {
1974 NEON::BI__builtin_neon_vabs_f16,
1975 NEON::BI__builtin_neon_vabs_v,
1976 },
1977 {
1978 NEON::BI__builtin_neon_vabsq_f16,
1979 NEON::BI__builtin_neon_vabsq_v,
1980 },
1981 {
1982 NEON::BI__builtin_neon_vcage_f16,
1983 NEON::BI__builtin_neon_vcage_v,
1984 },
1985 {
1986 NEON::BI__builtin_neon_vcageq_f16,
1987 NEON::BI__builtin_neon_vcageq_v,
1988 },
1989 {
1990 NEON::BI__builtin_neon_vcagt_f16,
1991 NEON::BI__builtin_neon_vcagt_v,
1992 },
1993 {
1994 NEON::BI__builtin_neon_vcagtq_f16,
1995 NEON::BI__builtin_neon_vcagtq_v,
1996 },
1997 {
1998 NEON::BI__builtin_neon_vcale_f16,
1999 NEON::BI__builtin_neon_vcale_v,
2000 },
2001 {
2002 NEON::BI__builtin_neon_vcaleq_f16,
2003 NEON::BI__builtin_neon_vcaleq_v,
2004 },
2005 {
2006 NEON::BI__builtin_neon_vcalt_f16,
2007 NEON::BI__builtin_neon_vcalt_v,
2008 },
2009 {
2010 NEON::BI__builtin_neon_vcaltq_f16,
2011 NEON::BI__builtin_neon_vcaltq_v,
2012 },
2013 {
2014 NEON::BI__builtin_neon_vceqz_f16,
2015 NEON::BI__builtin_neon_vceqz_v,
2016 },
2017 {
2018 NEON::BI__builtin_neon_vceqzq_f16,
2019 NEON::BI__builtin_neon_vceqzq_v,
2020 },
2021 {
2022 NEON::BI__builtin_neon_vcgez_f16,
2023 NEON::BI__builtin_neon_vcgez_v,
2024 },
2025 {
2026 NEON::BI__builtin_neon_vcgezq_f16,
2027 NEON::BI__builtin_neon_vcgezq_v,
2028 },
2029 {
2030 NEON::BI__builtin_neon_vcgtz_f16,
2031 NEON::BI__builtin_neon_vcgtz_v,
2032 },
2033 {
2034 NEON::BI__builtin_neon_vcgtzq_f16,
2035 NEON::BI__builtin_neon_vcgtzq_v,
2036 },
2037 {
2038 NEON::BI__builtin_neon_vclez_f16,
2039 NEON::BI__builtin_neon_vclez_v,
2040 },
2041 {
2042 NEON::BI__builtin_neon_vclezq_f16,
2043 NEON::BI__builtin_neon_vclezq_v,
2044 },
2045 {
2046 NEON::BI__builtin_neon_vcltz_f16,
2047 NEON::BI__builtin_neon_vcltz_v,
2048 },
2049 {
2050 NEON::BI__builtin_neon_vcltzq_f16,
2051 NEON::BI__builtin_neon_vcltzq_v,
2052 },
2053 {
2054 NEON::BI__builtin_neon_vfma_f16,
2055 NEON::BI__builtin_neon_vfma_v,
2056 },
2057 {
2058 NEON::BI__builtin_neon_vfma_lane_f16,
2059 NEON::BI__builtin_neon_vfma_lane_v,
2060 },
2061 {
2062 NEON::BI__builtin_neon_vfma_laneq_f16,
2063 NEON::BI__builtin_neon_vfma_laneq_v,
2064 },
2065 {
2066 NEON::BI__builtin_neon_vfmaq_f16,
2067 NEON::BI__builtin_neon_vfmaq_v,
2068 },
2069 {
2070 NEON::BI__builtin_neon_vfmaq_lane_f16,
2071 NEON::BI__builtin_neon_vfmaq_lane_v,
2072 },
2073 {
2074 NEON::BI__builtin_neon_vfmaq_laneq_f16,
2075 NEON::BI__builtin_neon_vfmaq_laneq_v,
2076 },
2077 {
2078 NEON::BI__builtin_neon_vmax_f16,
2079 NEON::BI__builtin_neon_vmax_v,
2080 },
2081 {
2082 NEON::BI__builtin_neon_vmaxnm_f16,
2083 NEON::BI__builtin_neon_vmaxnm_v,
2084 },
2085 {
2086 NEON::BI__builtin_neon_vmaxnmq_f16,
2087 NEON::BI__builtin_neon_vmaxnmq_v,
2088 },
2089 {
2090 NEON::BI__builtin_neon_vmaxq_f16,
2091 NEON::BI__builtin_neon_vmaxq_v,
2092 },
2093 {
2094 NEON::BI__builtin_neon_vmin_f16,
2095 NEON::BI__builtin_neon_vmin_v,
2096 },
2097 {
2098 NEON::BI__builtin_neon_vminnm_f16,
2099 NEON::BI__builtin_neon_vminnm_v,
2100 },
2101 {
2102 NEON::BI__builtin_neon_vminnmq_f16,
2103 NEON::BI__builtin_neon_vminnmq_v,
2104 },
2105 {
2106 NEON::BI__builtin_neon_vminq_f16,
2107 NEON::BI__builtin_neon_vminq_v,
2108 },
2109 {
2110 NEON::BI__builtin_neon_vmulx_f16,
2111 NEON::BI__builtin_neon_vmulx_v,
2112 },
2113 {
2114 NEON::BI__builtin_neon_vmulxq_f16,
2115 NEON::BI__builtin_neon_vmulxq_v,
2116 },
2117 {
2118 NEON::BI__builtin_neon_vpadd_f16,
2119 NEON::BI__builtin_neon_vpadd_v,
2120 },
2121 {
2122 NEON::BI__builtin_neon_vpaddq_f16,
2123 NEON::BI__builtin_neon_vpaddq_v,
2124 },
2125 {
2126 NEON::BI__builtin_neon_vpmax_f16,
2127 NEON::BI__builtin_neon_vpmax_v,
2128 },
2129 {
2130 NEON::BI__builtin_neon_vpmaxnm_f16,
2131 NEON::BI__builtin_neon_vpmaxnm_v,
2132 },
2133 {
2134 NEON::BI__builtin_neon_vpmaxnmq_f16,
2135 NEON::BI__builtin_neon_vpmaxnmq_v,
2136 },
2137 {
2138 NEON::BI__builtin_neon_vpmaxq_f16,
2139 NEON::BI__builtin_neon_vpmaxq_v,
2140 },
2141 {
2142 NEON::BI__builtin_neon_vpmin_f16,
2143 NEON::BI__builtin_neon_vpmin_v,
2144 },
2145 {
2146 NEON::BI__builtin_neon_vpminnm_f16,
2147 NEON::BI__builtin_neon_vpminnm_v,
2148 },
2149 {
2150 NEON::BI__builtin_neon_vpminnmq_f16,
2151 NEON::BI__builtin_neon_vpminnmq_v,
2152 },
2153 {
2154 NEON::BI__builtin_neon_vpminq_f16,
2155 NEON::BI__builtin_neon_vpminq_v,
2156 },
2157 {
2158 NEON::BI__builtin_neon_vrecpe_f16,
2159 NEON::BI__builtin_neon_vrecpe_v,
2160 },
2161 {
2162 NEON::BI__builtin_neon_vrecpeq_f16,
2163 NEON::BI__builtin_neon_vrecpeq_v,
2164 },
2165 {
2166 NEON::BI__builtin_neon_vrecps_f16,
2167 NEON::BI__builtin_neon_vrecps_v,
2168 },
2169 {
2170 NEON::BI__builtin_neon_vrecpsq_f16,
2171 NEON::BI__builtin_neon_vrecpsq_v,
2172 },
2173 {
2174 NEON::BI__builtin_neon_vrnd_f16,
2175 NEON::BI__builtin_neon_vrnd_v,
2176 },
2177 {
2178 NEON::BI__builtin_neon_vrnda_f16,
2179 NEON::BI__builtin_neon_vrnda_v,
2180 },
2181 {
2182 NEON::BI__builtin_neon_vrndaq_f16,
2183 NEON::BI__builtin_neon_vrndaq_v,
2184 },
2185 {
2186 NEON::BI__builtin_neon_vrndi_f16,
2187 NEON::BI__builtin_neon_vrndi_v,
2188 },
2189 {
2190 NEON::BI__builtin_neon_vrndiq_f16,
2191 NEON::BI__builtin_neon_vrndiq_v,
2192 },
2193 {
2194 NEON::BI__builtin_neon_vrndm_f16,
2195 NEON::BI__builtin_neon_vrndm_v,
2196 },
2197 {
2198 NEON::BI__builtin_neon_vrndmq_f16,
2199 NEON::BI__builtin_neon_vrndmq_v,
2200 },
2201 {
2202 NEON::BI__builtin_neon_vrndn_f16,
2203 NEON::BI__builtin_neon_vrndn_v,
2204 },
2205 {
2206 NEON::BI__builtin_neon_vrndnq_f16,
2207 NEON::BI__builtin_neon_vrndnq_v,
2208 },
2209 {
2210 NEON::BI__builtin_neon_vrndp_f16,
2211 NEON::BI__builtin_neon_vrndp_v,
2212 },
2213 {
2214 NEON::BI__builtin_neon_vrndpq_f16,
2215 NEON::BI__builtin_neon_vrndpq_v,
2216 },
2217 {
2218 NEON::BI__builtin_neon_vrndq_f16,
2219 NEON::BI__builtin_neon_vrndq_v,
2220 },
2221 {
2222 NEON::BI__builtin_neon_vrndx_f16,
2223 NEON::BI__builtin_neon_vrndx_v,
2224 },
2225 {
2226 NEON::BI__builtin_neon_vrndxq_f16,
2227 NEON::BI__builtin_neon_vrndxq_v,
2228 },
2229 {
2230 NEON::BI__builtin_neon_vrsqrte_f16,
2231 NEON::BI__builtin_neon_vrsqrte_v,
2232 },
2233 {
2234 NEON::BI__builtin_neon_vrsqrteq_f16,
2235 NEON::BI__builtin_neon_vrsqrteq_v,
2236 },
2237 {
2238 NEON::BI__builtin_neon_vrsqrts_f16,
2239 NEON::BI__builtin_neon_vrsqrts_v,
2240 },
2241 {
2242 NEON::BI__builtin_neon_vrsqrtsq_f16,
2243 NEON::BI__builtin_neon_vrsqrtsq_v,
2244 },
2245 {
2246 NEON::BI__builtin_neon_vsqrt_f16,
2247 NEON::BI__builtin_neon_vsqrt_v,
2248 },
2249 {
2250 NEON::BI__builtin_neon_vsqrtq_f16,
2251 NEON::BI__builtin_neon_vsqrtq_v,
2252 },
2253 // The mangling rules cause us to have one ID for each type for
2254 // vldap1(q)_lane and vstl1(q)_lane, but codegen is equivalent for all of
2255 // them. Choose an arbitrary one to be handled as tha canonical variation.
2256 {NEON::BI__builtin_neon_vldap1_lane_u64,
2257 NEON::BI__builtin_neon_vldap1_lane_s64},
2258 {NEON::BI__builtin_neon_vldap1_lane_f64,
2259 NEON::BI__builtin_neon_vldap1_lane_s64},
2260 {NEON::BI__builtin_neon_vldap1_lane_p64,
2261 NEON::BI__builtin_neon_vldap1_lane_s64},
2262 {NEON::BI__builtin_neon_vldap1q_lane_u64,
2263 NEON::BI__builtin_neon_vldap1q_lane_s64},
2264 {NEON::BI__builtin_neon_vldap1q_lane_f64,
2265 NEON::BI__builtin_neon_vldap1q_lane_s64},
2266 {NEON::BI__builtin_neon_vldap1q_lane_p64,
2267 NEON::BI__builtin_neon_vldap1q_lane_s64},
2268 {NEON::BI__builtin_neon_vstl1_lane_u64,
2269 NEON::BI__builtin_neon_vstl1_lane_s64},
2270 {NEON::BI__builtin_neon_vstl1_lane_f64,
2271 NEON::BI__builtin_neon_vstl1_lane_s64},
2272 {NEON::BI__builtin_neon_vstl1_lane_p64,
2273 NEON::BI__builtin_neon_vstl1_lane_s64},
2274 {NEON::BI__builtin_neon_vstl1q_lane_u64,
2275 NEON::BI__builtin_neon_vstl1q_lane_s64},
2276 {NEON::BI__builtin_neon_vstl1q_lane_f64,
2277 NEON::BI__builtin_neon_vstl1q_lane_s64},
2278 {NEON::BI__builtin_neon_vstl1q_lane_p64,
2279 NEON::BI__builtin_neon_vstl1q_lane_s64},
2280};
2281
2282std::optional<mlir::Value>
2283CIRGenFunction::emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr,
2284 ReturnValueSlot returnValue,
2285 llvm::Triple::ArchType arch) {
2286 if (builtinID >= clang::AArch64::FirstSVEBuiltin &&
2287 builtinID <= clang::AArch64::LastSVEBuiltin)
2288 return emitAArch64SVEBuiltinExpr(builtinID, expr);
2289
2290 if (builtinID >= clang::AArch64::FirstSMEBuiltin &&
2291 builtinID <= clang::AArch64::LastSMEBuiltin)
2292 return emitAArch64SMEBuiltinExpr(builtinID, expr);
2293
2294 if (builtinID == Builtin::BI__builtin_cpu_supports) {
2295 cgm.errorNYI(expr->getSourceRange(),
2296 std::string("unimplemented AArch64 builtin call: ") +
2297 getContext().BuiltinInfo.getName(ID: builtinID));
2298 return mlir::Value{};
2299 }
2300
2301 switch (builtinID) {
2302 default:
2303 break;
2304 case clang::AArch64::BI__builtin_arm_nop:
2305 case clang::AArch64::BI__builtin_arm_yield:
2306 case clang::AArch64::BI__yield:
2307 case clang::AArch64::BI__builtin_arm_wfe:
2308 case clang::AArch64::BI__wfe:
2309 case clang::AArch64::BI__builtin_arm_wfi:
2310 case clang::AArch64::BI__wfi:
2311 case clang::AArch64::BI__builtin_arm_sev:
2312 case clang::AArch64::BI__sev:
2313 case clang::AArch64::BI__builtin_arm_sevl:
2314 case clang::AArch64::BI__sevl:
2315 cgm.errorNYI(expr->getSourceRange(),
2316 std::string("unimplemented AArch64 builtin call: ") +
2317 getContext().BuiltinInfo.getName(ID: builtinID));
2318 return mlir::Value{};
2319 }
2320
2321 if (builtinID == clang::AArch64::BI__builtin_arm_trap) {
2322 cgm.errorNYI(expr->getSourceRange(),
2323 std::string("unimplemented AArch64 builtin call: ") +
2324 getContext().BuiltinInfo.getName(ID: builtinID));
2325 return mlir::Value{};
2326 }
2327
2328 if (builtinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
2329 cgm.errorNYI(expr->getSourceRange(),
2330 std::string("unimplemented AArch64 builtin call: ") +
2331 getContext().BuiltinInfo.getName(ID: builtinID));
2332 return mlir::Value{};
2333 }
2334
2335 if (builtinID == clang::AArch64::BI__builtin_arm_rbit) {
2336 cgm.errorNYI(expr->getSourceRange(),
2337 std::string("unimplemented AArch64 builtin call: ") +
2338 getContext().BuiltinInfo.getName(ID: builtinID));
2339 return mlir::Value{};
2340 }
2341 if (builtinID == clang::AArch64::BI__builtin_arm_rbit64) {
2342 cgm.errorNYI(expr->getSourceRange(),
2343 std::string("unimplemented AArch64 builtin call: ") +
2344 getContext().BuiltinInfo.getName(ID: builtinID));
2345 return mlir::Value{};
2346 }
2347
2348 if (builtinID == clang::AArch64::BI__builtin_arm_clz ||
2349 builtinID == clang::AArch64::BI__builtin_arm_clz64) {
2350 cgm.errorNYI(expr->getSourceRange(),
2351 std::string("unimplemented AArch64 builtin call: ") +
2352 getContext().BuiltinInfo.getName(ID: builtinID));
2353 return mlir::Value{};
2354 }
2355
2356 if (builtinID == clang::AArch64::BI__builtin_arm_cls) {
2357 cgm.errorNYI(expr->getSourceRange(),
2358 std::string("unimplemented AArch64 builtin call: ") +
2359 getContext().BuiltinInfo.getName(ID: builtinID));
2360 return mlir::Value{};
2361 }
2362 if (builtinID == clang::AArch64::BI__builtin_arm_cls64) {
2363 cgm.errorNYI(expr->getSourceRange(),
2364 std::string("unimplemented AArch64 builtin call: ") +
2365 getContext().BuiltinInfo.getName(ID: builtinID));
2366 return mlir::Value{};
2367 }
2368
2369 if (builtinID == clang::AArch64::BI__builtin_arm_rint32zf ||
2370 builtinID == clang::AArch64::BI__builtin_arm_rint32z) {
2371 cgm.errorNYI(expr->getSourceRange(),
2372 std::string("unimplemented AArch64 builtin call: ") +
2373 getContext().BuiltinInfo.getName(ID: builtinID));
2374 return mlir::Value{};
2375 }
2376
2377 if (builtinID == clang::AArch64::BI__builtin_arm_rint64zf ||
2378 builtinID == clang::AArch64::BI__builtin_arm_rint64z) {
2379 cgm.errorNYI(expr->getSourceRange(),
2380 std::string("unimplemented AArch64 builtin call: ") +
2381 getContext().BuiltinInfo.getName(ID: builtinID));
2382 return mlir::Value{};
2383 }
2384
2385 if (builtinID == clang::AArch64::BI__builtin_arm_rint32xf ||
2386 builtinID == clang::AArch64::BI__builtin_arm_rint32x) {
2387 cgm.errorNYI(expr->getSourceRange(),
2388 std::string("unimplemented AArch64 builtin call: ") +
2389 getContext().BuiltinInfo.getName(ID: builtinID));
2390 return mlir::Value{};
2391 }
2392
2393 if (builtinID == clang::AArch64::BI__builtin_arm_rint64xf ||
2394 builtinID == clang::AArch64::BI__builtin_arm_rint64x) {
2395 cgm.errorNYI(expr->getSourceRange(),
2396 std::string("unimplemented AArch64 builtin call: ") +
2397 getContext().BuiltinInfo.getName(ID: builtinID));
2398 return mlir::Value{};
2399 }
2400
2401 if (builtinID == clang::AArch64::BI__builtin_arm_jcvt) {
2402 cgm.errorNYI(expr->getSourceRange(),
2403 std::string("unimplemented AArch64 builtin call: ") +
2404 getContext().BuiltinInfo.getName(ID: builtinID));
2405 return mlir::Value{};
2406 }
2407
2408 if (builtinID == clang::AArch64::BI__builtin_arm_ld64b ||
2409 builtinID == clang::AArch64::BI__builtin_arm_st64b ||
2410 builtinID == clang::AArch64::BI__builtin_arm_st64bv ||
2411 builtinID == clang::AArch64::BI__builtin_arm_st64bv0) {
2412 cgm.errorNYI(expr->getSourceRange(),
2413 std::string("unimplemented AArch64 builtin call: ") +
2414 getContext().BuiltinInfo.getName(ID: builtinID));
2415 return mlir::Value{};
2416 }
2417
2418 if (builtinID == clang::AArch64::BI__builtin_arm_rndr ||
2419 builtinID == clang::AArch64::BI__builtin_arm_rndrrs) {
2420 cgm.errorNYI(expr->getSourceRange(),
2421 std::string("unimplemented AArch64 builtin call: ") +
2422 getContext().BuiltinInfo.getName(ID: builtinID));
2423 return mlir::Value{};
2424 }
2425
2426 if (builtinID == clang::AArch64::BI__clear_cache) {
2427 cgm.errorNYI(expr->getSourceRange(),
2428 std::string("unimplemented AArch64 builtin call: ") +
2429 getContext().BuiltinInfo.getName(ID: builtinID));
2430 return mlir::Value{};
2431 }
2432
2433 if ((builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
2434 builtinID == clang::AArch64::BI__builtin_arm_ldaex) &&
2435 getContext().getTypeSize(T: expr->getType()) == 128) {
2436 cgm.errorNYI(expr->getSourceRange(),
2437 std::string("unimplemented AArch64 builtin call: ") +
2438 getContext().BuiltinInfo.getName(ID: builtinID));
2439 return mlir::Value{};
2440 }
2441 if (builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
2442 builtinID == clang::AArch64::BI__builtin_arm_ldaex) {
2443 cgm.errorNYI(expr->getSourceRange(),
2444 std::string("unimplemented AArch64 builtin call: ") +
2445 getContext().BuiltinInfo.getName(ID: builtinID));
2446 return mlir::Value{};
2447 }
2448
2449 if ((builtinID == clang::AArch64::BI__builtin_arm_strex ||
2450 builtinID == clang::AArch64::BI__builtin_arm_stlex) &&
2451 getContext().getTypeSize(T: expr->getArg(Arg: 0)->getType()) == 128) {
2452 cgm.errorNYI(expr->getSourceRange(),
2453 std::string("unimplemented AArch64 builtin call: ") +
2454 getContext().BuiltinInfo.getName(ID: builtinID));
2455 return mlir::Value{};
2456 }
2457
2458 if (builtinID == clang::AArch64::BI__builtin_arm_strex ||
2459 builtinID == clang::AArch64::BI__builtin_arm_stlex) {
2460 cgm.errorNYI(expr->getSourceRange(),
2461 std::string("unimplemented AArch64 builtin call: ") +
2462 getContext().BuiltinInfo.getName(ID: builtinID));
2463 return mlir::Value{};
2464 }
2465
2466 if (builtinID == clang::AArch64::BI__getReg ||
2467 builtinID == clang::AArch64::BI__setReg ||
2468 builtinID == clang::AArch64::BI__getRegFp ||
2469 builtinID == clang::AArch64::BI__setRegFp) {
2470 cgm.errorNYI(expr->getSourceRange(),
2471 std::string("unimplemented AArch64 builtin call: ") +
2472 getContext().BuiltinInfo.getName(ID: builtinID));
2473 return mlir::Value{};
2474 }
2475
2476 if (builtinID == clang::AArch64::BI__break) {
2477 cgm.errorNYI(expr->getSourceRange(),
2478 std::string("unimplemented AArch64 builtin call: ") +
2479 getContext().BuiltinInfo.getName(ID: builtinID));
2480 return mlir::Value{};
2481 }
2482
2483 if (builtinID == clang::AArch64::BI__builtin_arm_clrex) {
2484 cgm.errorNYI(expr->getSourceRange(),
2485 std::string("unimplemented AArch64 builtin call: ") +
2486 getContext().BuiltinInfo.getName(ID: builtinID));
2487 return mlir::Value{};
2488 }
2489
2490 if (builtinID == clang::AArch64::BI_ReadWriteBarrier) {
2491 cgm.errorNYI(expr->getSourceRange(),
2492 std::string("unimplemented AArch64 builtin call: ") +
2493 getContext().BuiltinInfo.getName(ID: builtinID));
2494 return mlir::Value{};
2495 }
2496
2497 // CRC32
2498 Intrinsic::ID crcIntrinsicID = Intrinsic::not_intrinsic;
2499 switch (builtinID) {
2500 case clang::AArch64::BI__builtin_arm_crc32b:
2501 crcIntrinsicID = Intrinsic::aarch64_crc32b;
2502 break;
2503 case clang::AArch64::BI__builtin_arm_crc32cb:
2504 crcIntrinsicID = Intrinsic::aarch64_crc32cb;
2505 break;
2506 case clang::AArch64::BI__builtin_arm_crc32h:
2507 crcIntrinsicID = Intrinsic::aarch64_crc32h;
2508 break;
2509 case clang::AArch64::BI__builtin_arm_crc32ch:
2510 crcIntrinsicID = Intrinsic::aarch64_crc32ch;
2511 break;
2512 case clang::AArch64::BI__builtin_arm_crc32w:
2513 crcIntrinsicID = Intrinsic::aarch64_crc32w;
2514 break;
2515 case clang::AArch64::BI__builtin_arm_crc32cw:
2516 crcIntrinsicID = Intrinsic::aarch64_crc32cw;
2517 break;
2518 case clang::AArch64::BI__builtin_arm_crc32d:
2519 crcIntrinsicID = Intrinsic::aarch64_crc32x;
2520 break;
2521 case clang::AArch64::BI__builtin_arm_crc32cd:
2522 crcIntrinsicID = Intrinsic::aarch64_crc32cx;
2523 break;
2524 }
2525
2526 if (crcIntrinsicID != Intrinsic::not_intrinsic) {
2527 cgm.errorNYI(expr->getSourceRange(),
2528 std::string("unimplemented AArch64 builtin call: ") +
2529 getContext().BuiltinInfo.getName(ID: builtinID));
2530 return mlir::Value{};
2531 }
2532
2533 // Memory Operations (MOPS)
2534 if (builtinID == AArch64::BI__builtin_arm_mops_memset_tag) {
2535 cgm.errorNYI(expr->getSourceRange(),
2536 std::string("unimplemented AArch64 builtin call: ") +
2537 getContext().BuiltinInfo.getName(ID: builtinID));
2538 return mlir::Value{};
2539 }
2540
2541 // Memory Tagging Extensions (MTE) Intrinsics
2542 Intrinsic::ID mteIntrinsicID = Intrinsic::not_intrinsic;
2543 switch (builtinID) {
2544 case clang::AArch64::BI__builtin_arm_irg:
2545 mteIntrinsicID = Intrinsic::aarch64_irg;
2546 break;
2547 case clang::AArch64::BI__builtin_arm_addg:
2548 mteIntrinsicID = Intrinsic::aarch64_addg;
2549 break;
2550 case clang::AArch64::BI__builtin_arm_gmi:
2551 mteIntrinsicID = Intrinsic::aarch64_gmi;
2552 break;
2553 case clang::AArch64::BI__builtin_arm_ldg:
2554 mteIntrinsicID = Intrinsic::aarch64_ldg;
2555 break;
2556 case clang::AArch64::BI__builtin_arm_stg:
2557 mteIntrinsicID = Intrinsic::aarch64_stg;
2558 break;
2559 case clang::AArch64::BI__builtin_arm_subp:
2560 mteIntrinsicID = Intrinsic::aarch64_subp;
2561 break;
2562 }
2563
2564 if (mteIntrinsicID != Intrinsic::not_intrinsic) {
2565 cgm.errorNYI(expr->getSourceRange(),
2566 std::string("unimplemented AArch64 builtin call: ") +
2567 getContext().BuiltinInfo.getName(ID: builtinID));
2568 return mlir::Value{};
2569 }
2570
2571 if (builtinID == clang::AArch64::BI__builtin_arm_rsr ||
2572 builtinID == clang::AArch64::BI__builtin_arm_rsr64 ||
2573 builtinID == clang::AArch64::BI__builtin_arm_rsr128 ||
2574 builtinID == clang::AArch64::BI__builtin_arm_rsrp ||
2575 builtinID == clang::AArch64::BI__builtin_arm_wsr ||
2576 builtinID == clang::AArch64::BI__builtin_arm_wsr64 ||
2577 builtinID == clang::AArch64::BI__builtin_arm_wsr128 ||
2578 builtinID == clang::AArch64::BI__builtin_arm_wsrp) {
2579 cgm.errorNYI(expr->getSourceRange(),
2580 std::string("unimplemented AArch64 builtin call: ") +
2581 getContext().BuiltinInfo.getName(ID: builtinID));
2582 return mlir::Value{};
2583 }
2584
2585 if (builtinID == clang::AArch64::BI_ReadStatusReg ||
2586 builtinID == clang::AArch64::BI_WriteStatusReg ||
2587 builtinID == clang::AArch64::BI__sys) {
2588 cgm.errorNYI(expr->getSourceRange(),
2589 std::string("unimplemented AArch64 builtin call: ") +
2590 getContext().BuiltinInfo.getName(ID: builtinID));
2591 return mlir::Value{};
2592 }
2593
2594 if (builtinID == clang::AArch64::BI_AddressOfReturnAddress) {
2595 cgm.errorNYI(expr->getSourceRange(),
2596 std::string("unimplemented AArch64 builtin call: ") +
2597 getContext().BuiltinInfo.getName(ID: builtinID));
2598 return mlir::Value{};
2599 }
2600
2601 if (builtinID == clang::AArch64::BI__builtin_sponentry) {
2602 cgm.errorNYI(expr->getSourceRange(),
2603 std::string("unimplemented AArch64 builtin call: ") +
2604 getContext().BuiltinInfo.getName(ID: builtinID));
2605 return mlir::Value{};
2606 }
2607
2608 if (builtinID == clang::AArch64::BI__mulh ||
2609 builtinID == clang::AArch64::BI__umulh) {
2610 cgm.errorNYI(expr->getSourceRange(),
2611 std::string("unimplemented AArch64 builtin call: ") +
2612 getContext().BuiltinInfo.getName(ID: builtinID));
2613 return mlir::Value{};
2614 }
2615
2616 if (builtinID == AArch64::BI__writex18byte ||
2617 builtinID == AArch64::BI__writex18word ||
2618 builtinID == AArch64::BI__writex18dword ||
2619 builtinID == AArch64::BI__writex18qword) {
2620 cgm.errorNYI(expr->getSourceRange(),
2621 std::string("unimplemented AArch64 builtin call: ") +
2622 getContext().BuiltinInfo.getName(ID: builtinID));
2623 return mlir::Value{};
2624 }
2625
2626 if (builtinID == AArch64::BI__readx18byte ||
2627 builtinID == AArch64::BI__readx18word ||
2628 builtinID == AArch64::BI__readx18dword ||
2629 builtinID == AArch64::BI__readx18qword) {
2630 cgm.errorNYI(expr->getSourceRange(),
2631 std::string("unimplemented AArch64 builtin call: ") +
2632 getContext().BuiltinInfo.getName(ID: builtinID));
2633 return mlir::Value{};
2634 }
2635
2636 if (builtinID == AArch64::BI__addx18byte ||
2637 builtinID == AArch64::BI__addx18word ||
2638 builtinID == AArch64::BI__addx18dword ||
2639 builtinID == AArch64::BI__addx18qword ||
2640 builtinID == AArch64::BI__incx18byte ||
2641 builtinID == AArch64::BI__incx18word ||
2642 builtinID == AArch64::BI__incx18dword ||
2643 builtinID == AArch64::BI__incx18qword) {
2644 cgm.errorNYI(expr->getSourceRange(),
2645 std::string("unimplemented AArch64 builtin call: ") +
2646 getContext().BuiltinInfo.getName(ID: builtinID));
2647 return mlir::Value{};
2648 }
2649
2650 if (builtinID == AArch64::BI_CopyDoubleFromInt64 ||
2651 builtinID == AArch64::BI_CopyFloatFromInt32 ||
2652 builtinID == AArch64::BI_CopyInt32FromFloat ||
2653 builtinID == AArch64::BI_CopyInt64FromDouble) {
2654 cgm.errorNYI(expr->getSourceRange(),
2655 std::string("unimplemented AArch64 builtin call: ") +
2656 getContext().BuiltinInfo.getName(ID: builtinID));
2657 return mlir::Value{};
2658 }
2659
2660 if (builtinID == AArch64::BI_CountLeadingOnes ||
2661 builtinID == AArch64::BI_CountLeadingOnes64 ||
2662 builtinID == AArch64::BI_CountLeadingZeros ||
2663 builtinID == AArch64::BI_CountLeadingZeros64) {
2664 cgm.errorNYI(expr->getSourceRange(),
2665 std::string("unimplemented AArch64 builtin call: ") +
2666 getContext().BuiltinInfo.getName(ID: builtinID));
2667 return mlir::Value{};
2668 }
2669
2670 if (builtinID == AArch64::BI_CountLeadingSigns ||
2671 builtinID == AArch64::BI_CountLeadingSigns64) {
2672 cgm.errorNYI(expr->getSourceRange(),
2673 std::string("unimplemented AArch64 builtin call: ") +
2674 getContext().BuiltinInfo.getName(ID: builtinID));
2675 return mlir::Value{};
2676 }
2677
2678 if (builtinID == AArch64::BI_CountOneBits ||
2679 builtinID == AArch64::BI_CountOneBits64 ||
2680 builtinID == AArch64::BI_CountTrailingZeros ||
2681 builtinID == AArch64::BI_CountTrailingZeros64) {
2682 cgm.errorNYI(expr->getSourceRange(),
2683 std::string("unimplemented AArch64 builtin call: ") +
2684 getContext().BuiltinInfo.getName(ID: builtinID));
2685 return mlir::Value{};
2686 }
2687
2688 if (builtinID == AArch64::BI__prefetch ||
2689 builtinID == AArch64::BI__prefetch2) {
2690 cgm.errorNYI(expr->getSourceRange(),
2691 std::string("unimplemented AArch64 builtin call: ") +
2692 getContext().BuiltinInfo.getName(ID: builtinID));
2693 return mlir::Value{};
2694 }
2695
2696 if (builtinID == AArch64::BI__hlt) {
2697 cgm.errorNYI(expr->getSourceRange(),
2698 std::string("unimplemented AArch64 builtin call: ") +
2699 getContext().BuiltinInfo.getName(ID: builtinID));
2700 return mlir::Value{};
2701 }
2702
2703 if (builtinID == NEON::BI__builtin_neon_vcvth_bf16_f32) {
2704 cgm.errorNYI(expr->getSourceRange(),
2705 std::string("unimplemented AArch64 builtin call: ") +
2706 getContext().BuiltinInfo.getName(ID: builtinID));
2707 return mlir::Value{};
2708 }
2709
2710 // Handle MSVC intrinsics before argument evaluation to prevent double
2711 // evaluation.
2712 assert(!cir::MissingFeatures::msvcBuiltins());
2713
2714 CIRGenFPOptionsRAII fpOptsRAII(*this, expr);
2715
2716 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
2717 auto it = llvm::find_if(Range: neonEquivalentIntrinsicMap, P: [builtinID](auto &p) {
2718 return p.first == builtinID;
2719 });
2720 if (it != end(arr: neonEquivalentIntrinsicMap))
2721 builtinID = it->second;
2722
2723 // Find out if any arguments are required to be integer constant
2724 // expressions.
2725 assert(!cir::MissingFeatures::handleBuiltinICEArguments());
2726 unsigned iceArguments = 0;
2727 ASTContext::GetBuiltinTypeError error;
2728 getContext().GetBuiltinType(ID: builtinID, Error&: error, IntegerConstantArgs: &iceArguments);
2729 assert(error == ASTContext::GE_None && "Should not codegen an error");
2730 llvm::SmallVector<mlir::Value> ops;
2731
2732 // Captures the pointer and its alignment for builtins that store/load
2733 // directly through the first argument, i.e. operand 0 (set in the
2734 // arg-gathering loop below, used later when lowering the store/load
2735 // itself).
2736 Address ptrOp0 = Address::invalid();
2737
2738 // Skip extra arguments used to discriminate vector types and that are
2739 // intended for Sema checking.
2740 bool hasExtraArg = CodeGenUtils::hasExtraNeonArgument(BuiltinID: builtinID);
2741 unsigned numArgs = expr->getNumArgs() - (hasExtraArg ? 1 : 0);
2742 for (unsigned i = 0, e = numArgs; i != e; i++) {
2743 if (i == 0) {
2744 switch (builtinID) {
2745 case NEON::BI__builtin_neon_vld1_v:
2746 case NEON::BI__builtin_neon_vld1q_v:
2747 case NEON::BI__builtin_neon_vld1_dup_v:
2748 case NEON::BI__builtin_neon_vld1q_dup_v:
2749 case NEON::BI__builtin_neon_vld1_lane_v:
2750 case NEON::BI__builtin_neon_vld1q_lane_v:
2751 case NEON::BI__builtin_neon_vst1_v:
2752 case NEON::BI__builtin_neon_vst1q_v:
2753 case NEON::BI__builtin_neon_vst1_lane_v:
2754 case NEON::BI__builtin_neon_vst1q_lane_v:
2755 case NEON::BI__builtin_neon_vstrq_p128:
2756 case NEON::BI__builtin_neon_vst2_v:
2757 case NEON::BI__builtin_neon_vst2q_v:
2758 case NEON::BI__builtin_neon_vst2_lane_v:
2759 case NEON::BI__builtin_neon_vst2q_lane_v:
2760 case NEON::BI__builtin_neon_vst3_v:
2761 case NEON::BI__builtin_neon_vst3q_v:
2762 case NEON::BI__builtin_neon_vst3_lane_v:
2763 case NEON::BI__builtin_neon_vst3q_lane_v:
2764 case NEON::BI__builtin_neon_vst4_v:
2765 case NEON::BI__builtin_neon_vst4q_v:
2766 case NEON::BI__builtin_neon_vst4_lane_v:
2767 case NEON::BI__builtin_neon_vst4q_lane_v:
2768 // Get the alignment for the argument in addition to the value;
2769 // we'll use it later.
2770 ptrOp0 = emitPointerWithAlignment(expr: expr->getArg(Arg: 0));
2771 ops.push_back(Elt: ptrOp0.getPointer());
2772 continue;
2773 case NEON::BI__builtin_neon_vldap1_lane_s64:
2774 case NEON::BI__builtin_neon_vldap1q_lane_s64:
2775 case NEON::BI__builtin_neon_vstl1_lane_s64:
2776 case NEON::BI__builtin_neon_vstl1q_lane_s64:
2777 cgm.errorNYI(
2778 expr->getSourceRange(),
2779 std::string("unimplemented AArch64 builtin argument handling ") +
2780 getContext().BuiltinInfo.getName(ID: builtinID));
2781 break;
2782 }
2783 }
2784 ops.push_back(
2785 Elt: emitScalarOrConstFoldImmArg(iceArguments, idx: i, argExpr: expr->getArg(Arg: i)));
2786 }
2787
2788 const ARMNeonVectorIntrinsicInfo *builtin =
2789 findARMVectorIntrinsicInMap(intrinsicMap: ArrayRef(AArch64SISDIntrinsicMap), builtinID,
2790 mapProvenSorted&: aarch64SISDIntrinsicsProvenSorted);
2791 if (builtin)
2792 return emitCommonNeonSISDBuiltinExpr(cgf&: *this, info: *builtin, ops, expr,
2793 iceArguments);
2794
2795 // Not all intrinsics handled by the common case work for AArch64 yet, so only
2796 // defer to common code if it's been added to our special map.
2797 assert(!cir::MissingFeatures::aarch64SIMDIntrinsics());
2798
2799 assert(!cir::MissingFeatures::aarch64TblBuiltinExpr());
2800
2801 const Expr *arg = expr->getArg(Arg: expr->getNumArgs() - 1);
2802 NeonTypeFlags type(0);
2803 // A trailing constant integer is used for discriminating overloaded builtin
2804 // calls. Use it to determine the type of this overloaded NEON intrinsic.
2805 if (std::optional<llvm::APSInt> result =
2806 arg->getIntegerConstantExpr(Ctx: getContext()))
2807 type = NeonTypeFlags(result->getZExtValue());
2808
2809 bool usgn = type.isUnsigned();
2810
2811 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
2812
2813 // Not all intrinsics handled by the common case work for AArch64 yet, so only
2814 // defer to common code if it's been added to our special map.
2815 builtin =
2816 findARMVectorIntrinsicInMap(intrinsicMap: ArrayRef(AArch64SIMDIntrinsicMap), builtinID,
2817 mapProvenSorted&: aarch64SIMDIntrinsicsProvenSorted);
2818 if (builtin)
2819 return emitCommonNeonBuiltinExpr(
2820 cgf&: *this, builtinID: builtin->BuiltinID, llvmIntrinsic: builtin->LLVMIntrinsic,
2821 altLLVMIntrinsic: builtin->AltLLVMIntrinsic, nameHint: builtin->NameHint, modifier: builtin->TypeModifier,
2822 expr, ops);
2823
2824 // Handle non-overloaded intrinsics first.
2825 switch (builtinID) {
2826 default:
2827 break;
2828 case NEON::BI__builtin_neon_vabsh_f16: {
2829 mlir::Type halfTy = builder.getFp16Ty();
2830 return emitNeonCallToOp<cir::FAbsOp>(cgm, builder, argTypes: {halfTy}, args&: ops,
2831 intrinsicName: std::nullopt, funcResTy: halfTy, loc);
2832 }
2833 case NEON::BI__builtin_neon_vaddq_p128: {
2834 cir::VectorType byteTy =
2835 cir::VectorType::getFixed(elementType: builder.getUInt8Ty(), size: 16);
2836 ops[0] = builder.createBitcast(src: ops[0], newTy: byteTy);
2837 ops[1] = builder.createBitcast(src: ops[1], newTy: byteTy);
2838 mlir::Value result = builder.createXor(loc, lhs: ops[0], rhs: ops[1]);
2839 return builder.createBitcast(src: result, newTy: convertType(t: expr->getType()));
2840 }
2841 case NEON::BI__builtin_neon_vldrq_p128:
2842 cgm.errorNYI(expr->getSourceRange(),
2843 std::string("unimplemented AArch64 builtin call: ") +
2844 getContext().BuiltinInfo.getName(ID: builtinID));
2845 return mlir::Value{};
2846 case NEON::BI__builtin_neon_vstrq_p128: {
2847 builder.createStore(loc, val: ops[1], dst: ptrOp0);
2848 return nullptr;
2849 }
2850 case NEON::BI__builtin_neon_vcvts_f32_u32:
2851 case NEON::BI__builtin_neon_vcvtd_f64_u64:
2852 case NEON::BI__builtin_neon_vcvts_f32_s32:
2853 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
2854 return builder.createCast(loc, kind: cir::CastKind::int_to_float, src: ops[0],
2855 newTy: convertType(t: expr->getType()));
2856 }
2857 case NEON::BI__builtin_neon_vcvth_f16_u16:
2858 case NEON::BI__builtin_neon_vcvth_f16_u32:
2859 case NEON::BI__builtin_neon_vcvth_f16_u64:
2860 case NEON::BI__builtin_neon_vcvth_f16_s16:
2861 case NEON::BI__builtin_neon_vcvth_f16_s32:
2862 case NEON::BI__builtin_neon_vcvth_f16_s64:
2863 case NEON::BI__builtin_neon_vcvtah_u16_f16:
2864 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
2865 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
2866 case NEON::BI__builtin_neon_vcvtph_u16_f16:
2867 case NEON::BI__builtin_neon_vcvth_u16_f16:
2868 case NEON::BI__builtin_neon_vcvtah_s16_f16:
2869 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
2870 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
2871 case NEON::BI__builtin_neon_vcvtph_s16_f16:
2872 case NEON::BI__builtin_neon_vcvth_s16_f16:
2873 case NEON::BI__builtin_neon_vcaleh_f16:
2874 case NEON::BI__builtin_neon_vcalth_f16:
2875 case NEON::BI__builtin_neon_vcageh_f16:
2876 case NEON::BI__builtin_neon_vcagth_f16:
2877 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
2878 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
2879 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
2880 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
2881 case NEON::BI__builtin_neon_vpaddd_s64:
2882 case NEON::BI__builtin_neon_vpaddd_f64:
2883 case NEON::BI__builtin_neon_vpadds_f32:
2884 cgm.errorNYI(expr->getSourceRange(),
2885 std::string("unimplemented AArch64 builtin call: ") +
2886 getContext().BuiltinInfo.getName(ID: builtinID));
2887 return mlir::Value{};
2888 case NEON::BI__builtin_neon_vceqzd_s64:
2889 case NEON::BI__builtin_neon_vceqzd_f64:
2890 case NEON::BI__builtin_neon_vceqzs_f32:
2891 case NEON::BI__builtin_neon_vceqzh_f16:
2892 return emitAArch64CompareBuiltinExpr(
2893 cgf&: *this, builder, loc, src: ops[0],
2894 retTy: convertType(t: expr->getCallReturnType(Ctx: getContext())), kind: cir::CmpOpKind::eq);
2895 case NEON::BI__builtin_neon_vcgezd_s64:
2896 case NEON::BI__builtin_neon_vcgezd_f64:
2897 case NEON::BI__builtin_neon_vcgezs_f32:
2898 case NEON::BI__builtin_neon_vcgezh_f16:
2899 case NEON::BI__builtin_neon_vclezd_s64:
2900 case NEON::BI__builtin_neon_vclezd_f64:
2901 case NEON::BI__builtin_neon_vclezs_f32:
2902 case NEON::BI__builtin_neon_vclezh_f16:
2903 case NEON::BI__builtin_neon_vcgtzd_s64:
2904 case NEON::BI__builtin_neon_vcgtzd_f64:
2905 case NEON::BI__builtin_neon_vcgtzs_f32:
2906 case NEON::BI__builtin_neon_vcgtzh_f16:
2907 case NEON::BI__builtin_neon_vcltzd_s64:
2908 case NEON::BI__builtin_neon_vcltzd_f64:
2909 case NEON::BI__builtin_neon_vcltzs_f32:
2910 case NEON::BI__builtin_neon_vcltzh_f16:
2911 case NEON::BI__builtin_neon_vceqzd_u64: {
2912 return emitAArch64CompareBuiltinExpr(
2913 cgf&: *this, builder, loc, src: ops[0],
2914 retTy: convertType(t: expr->getCallReturnType(Ctx: getContext())), kind: cir::CmpOpKind::eq);
2915 }
2916 case NEON::BI__builtin_neon_vceqd_f64:
2917 case NEON::BI__builtin_neon_vcled_f64:
2918 case NEON::BI__builtin_neon_vcltd_f64:
2919 case NEON::BI__builtin_neon_vcged_f64:
2920 case NEON::BI__builtin_neon_vcgtd_f64:
2921 case NEON::BI__builtin_neon_vceqs_f32:
2922 case NEON::BI__builtin_neon_vcles_f32:
2923 case NEON::BI__builtin_neon_vclts_f32:
2924 case NEON::BI__builtin_neon_vcges_f32:
2925 case NEON::BI__builtin_neon_vcgts_f32:
2926 case NEON::BI__builtin_neon_vceqh_f16:
2927 case NEON::BI__builtin_neon_vcleh_f16:
2928 case NEON::BI__builtin_neon_vclth_f16:
2929 case NEON::BI__builtin_neon_vcgeh_f16:
2930 case NEON::BI__builtin_neon_vcgth_f16:
2931 case NEON::BI__builtin_neon_vceqd_s64:
2932 case NEON::BI__builtin_neon_vceqd_u64:
2933 case NEON::BI__builtin_neon_vcgtd_s64:
2934 case NEON::BI__builtin_neon_vcgtd_u64:
2935 case NEON::BI__builtin_neon_vcltd_s64:
2936 case NEON::BI__builtin_neon_vcltd_u64:
2937 case NEON::BI__builtin_neon_vcged_u64:
2938 case NEON::BI__builtin_neon_vcged_s64:
2939 case NEON::BI__builtin_neon_vcled_u64:
2940 case NEON::BI__builtin_neon_vcled_s64:
2941 cgm.errorNYI(expr->getSourceRange(),
2942 std::string("unimplemented AArch64 builtin call: ") +
2943 getContext().BuiltinInfo.getName(ID: builtinID));
2944 return mlir::Value{};
2945 case NEON::BI__builtin_neon_vnegd_s64: {
2946 return builder.createNeg(loc, value: ops[0]);
2947 }
2948 case NEON::BI__builtin_neon_vnegh_f16: {
2949 return builder.createFNeg(loc, operand: ops[0]);
2950 }
2951 case NEON::BI__builtin_neon_vtstd_s64:
2952 case NEON::BI__builtin_neon_vtstd_u64:
2953 cgm.errorNYI(expr->getSourceRange(),
2954 std::string("unimplemented AArch64 builtin call: ") +
2955 getContext().BuiltinInfo.getName(ID: builtinID));
2956 return mlir::Value{};
2957 case NEON::BI__builtin_neon_vset_lane_i8:
2958 case NEON::BI__builtin_neon_vset_lane_i16:
2959 case NEON::BI__builtin_neon_vset_lane_i32:
2960 case NEON::BI__builtin_neon_vset_lane_i64:
2961 case NEON::BI__builtin_neon_vset_lane_bf16:
2962 case NEON::BI__builtin_neon_vset_lane_f32:
2963 case NEON::BI__builtin_neon_vsetq_lane_i8:
2964 case NEON::BI__builtin_neon_vsetq_lane_i16:
2965 case NEON::BI__builtin_neon_vsetq_lane_i32:
2966 case NEON::BI__builtin_neon_vsetq_lane_i64:
2967 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2968 case NEON::BI__builtin_neon_vsetq_lane_f32:
2969 case NEON::BI__builtin_neon_vset_lane_f64:
2970 case NEON::BI__builtin_neon_vset_lane_mf8:
2971 case NEON::BI__builtin_neon_vsetq_lane_mf8:
2972 case NEON::BI__builtin_neon_vsetq_lane_f64:
2973 return cir::VecInsertOp::create(builder, location: loc, vec: ops[1], value: ops[0], index: ops[2]);
2974 case NEON::BI__builtin_neon_vget_lane_i8:
2975 case NEON::BI__builtin_neon_vdupb_lane_i8:
2976 case NEON::BI__builtin_neon_vgetq_lane_i8:
2977 case NEON::BI__builtin_neon_vdupb_laneq_i8:
2978 case NEON::BI__builtin_neon_vget_lane_mf8:
2979 case NEON::BI__builtin_neon_vdupb_lane_mf8:
2980 case NEON::BI__builtin_neon_vgetq_lane_mf8:
2981 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
2982 case NEON::BI__builtin_neon_vget_lane_i16:
2983 case NEON::BI__builtin_neon_vduph_lane_i16:
2984 case NEON::BI__builtin_neon_vgetq_lane_i16:
2985 case NEON::BI__builtin_neon_vduph_laneq_i16:
2986 case NEON::BI__builtin_neon_vget_lane_i32:
2987 case NEON::BI__builtin_neon_vdups_lane_i32:
2988 case NEON::BI__builtin_neon_vdups_lane_f32:
2989 case NEON::BI__builtin_neon_vgetq_lane_i32:
2990 case NEON::BI__builtin_neon_vdups_laneq_i32:
2991 case NEON::BI__builtin_neon_vget_lane_i64:
2992 case NEON::BI__builtin_neon_vdupd_lane_i64:
2993 case NEON::BI__builtin_neon_vdupd_lane_f64:
2994 case NEON::BI__builtin_neon_vgetq_lane_i64:
2995 case NEON::BI__builtin_neon_vdupd_laneq_i64:
2996 case NEON::BI__builtin_neon_vget_lane_f32:
2997 case NEON::BI__builtin_neon_vget_lane_f64:
2998 case NEON::BI__builtin_neon_vgetq_lane_f32:
2999 case NEON::BI__builtin_neon_vdups_laneq_f32:
3000 case NEON::BI__builtin_neon_vgetq_lane_f64:
3001 case NEON::BI__builtin_neon_vdupd_laneq_f64:
3002 return cir::VecExtractOp::create(builder, location: loc, vec: ops[0],
3003 index: emitScalarExpr(e: expr->getArg(Arg: 1)));
3004 case NEON::BI__builtin_neon_vaddh_f16:
3005 return builder.createFAdd(loc, lhs: ops[0], rhs: ops[1]);
3006 case NEON::BI__builtin_neon_vsubh_f16:
3007 return builder.createFSub(loc, lhs: ops[0], rhs: ops[1]);
3008 case NEON::BI__builtin_neon_vmulh_f16:
3009 return builder.createFMul(loc, lhs: ops[0], rhs: ops[1]);
3010 case NEON::BI__builtin_neon_vdivh_f16:
3011 return builder.createFDiv(loc, lhs: ops[0], rhs: ops[1]);
3012 case NEON::BI__builtin_neon_vfmah_f16: {
3013 // NEON intrinsic puts accumulator first, unlike fma.
3014 std::rotate(first: ops.begin(), middle: ops.begin() + 1, last: ops.end());
3015 mlir::Type ty = convertType(t: expr->getType());
3016 return emitNeonCallToOp<cir::FMAOp>(cgm, builder, argTypes: {ty, ty, ty}, args&: ops,
3017 intrinsicName: std::nullopt, funcResTy: ty, loc);
3018 }
3019 case NEON::BI__builtin_neon_vfmsh_f16: {
3020 // NEON intrinsic puts accumulator first, unlike fma.
3021 std::rotate(first: ops.begin(), middle: ops.begin() + 1, last: ops.end());
3022 ops[0] = builder.createFNeg(loc, operand: ops[0]);
3023 mlir::Type ty = convertType(t: expr->getType());
3024 return emitNeonCallToOp<cir::FMAOp>(cgm, builder, argTypes: {ty, ty, ty}, args&: ops,
3025 intrinsicName: std::nullopt, funcResTy: ty, loc);
3026 }
3027 case NEON::BI__builtin_neon_vaddd_s64:
3028 case NEON::BI__builtin_neon_vaddd_u64:
3029 return builder.createAdd(loc, lhs: ops[0], rhs: ops[1]);
3030 case NEON::BI__builtin_neon_vsubd_s64:
3031 case NEON::BI__builtin_neon_vsubd_u64:
3032 return builder.createSub(loc, lhs: ops[0], rhs: ops[1]);
3033 case NEON::BI__builtin_neon_vqdmlalh_s16:
3034 case NEON::BI__builtin_neon_vqdmlslh_s16:
3035 cgm.errorNYI(expr->getSourceRange(),
3036 std::string("unimplemented AArch64 builtin call: ") +
3037 getContext().BuiltinInfo.getName(ID: builtinID));
3038 return mlir::Value{};
3039 case NEON::BI__builtin_neon_vqshlud_n_s64: {
3040 cir::IntType int64Type = builder.getSInt64Ty();
3041 ops[1] = builder.getSInt64(c: getZExtIntValueFromConstOp(val: ops[1]), loc);
3042 return emitNeonCall(cgm, builder, argTypes: {int64Type, int64Type}, args&: ops,
3043 intrinsicName: "aarch64.neon.sqshlu", funcResTy: convertType(t: expr->getType()),
3044 loc);
3045 }
3046 case NEON::BI__builtin_neon_vqshld_n_u64:
3047 case NEON::BI__builtin_neon_vqshld_n_s64: {
3048 cir::IntType int64Type = builtinID == NEON::BI__builtin_neon_vqshld_n_u64
3049 ? builder.getUInt64Ty()
3050 : builder.getSInt64Ty();
3051 llvm::StringRef intrinsicName =
3052 builtinID == NEON::BI__builtin_neon_vqshld_n_u64 ? "aarch64.neon.uqshl"
3053 : "aarch64.neon.sqshl";
3054 ops[1] = builder.getSInt64(c: getZExtIntValueFromConstOp(val: ops[1]), loc);
3055 return emitNeonCall(cgm, builder, argTypes: {int64Type, int64Type}, args&: ops,
3056 intrinsicName, funcResTy: convertType(t: expr->getType()), loc);
3057 }
3058 case NEON::BI__builtin_neon_vrshrd_n_u64:
3059 case NEON::BI__builtin_neon_vrshrd_n_s64: {
3060 llvm::StringRef intrName = builtinID == NEON::BI__builtin_neon_vrshrd_n_s64
3061 ? "aarch64.neon.srshl"
3062 : "aarch64.neon.urshl";
3063 cir::IntType int64Ty = builtinID == NEON::BI__builtin_neon_vqshld_n_u64
3064 ? builder.getUInt64Ty()
3065 : builder.getSInt64Ty();
3066 int64_t sv = -cast<cir::IntAttr>(
3067 Val: cast<cir::ConstantOp>(Val: ops[1].getDefiningOp()).getValue())
3068 .getSInt();
3069 ops[1] = builder.getSInt64(c: sv, loc);
3070 return emitNeonCall(cgm, builder, argTypes: {int64Ty, builder.getSInt64Ty()}, args&: ops,
3071 intrinsicName: intrName, funcResTy: int64Ty, loc);
3072 }
3073 case NEON::BI__builtin_neon_vrsrad_n_u64:
3074 case NEON::BI__builtin_neon_vrsrad_n_s64: {
3075 cir::IntType int64Type = builtinID == NEON::BI__builtin_neon_vrsrad_n_u64
3076 ? builder.getUInt64Ty()
3077 : builder.getSInt64Ty();
3078 ops[2] = builder.createNeg(loc, value: ops[2]);
3079 const StringRef intrName = builtinID == NEON::BI__builtin_neon_vrsrad_n_u64
3080 ? "aarch64.neon.urshl"
3081 : "aarch64.neon.srshl";
3082
3083 llvm::SmallVector<mlir::Value, 2> args = {
3084 ops[1], builder.createIntCast(src: ops[2], newTy: builder.getSInt64Ty())};
3085 ops[1] = builder.emitIntrinsicCallOp(loc, str: intrName, resTy: int64Type, op&: args);
3086 return builder.createAdd(loc, lhs: ops[0],
3087 rhs: builder.createBitcast(src: ops[1], newTy: int64Type));
3088 }
3089 case NEON::BI__builtin_neon_vshld_n_s64:
3090 case NEON::BI__builtin_neon_vshld_n_u64: {
3091 auto loc = getLoc(srcLoc: expr->getExprLoc());
3092 std::optional<llvm::APSInt> amt =
3093 expr->getArg(Arg: 1)->getIntegerConstantExpr(Ctx: getContext());
3094 assert(amt && "Expected argument to be a constant");
3095 return builder.createShiftLeft(loc, lhs: ops[0], bits: amt->getZExtValue());
3096 }
3097 case NEON::BI__builtin_neon_vshrd_n_s64: {
3098 std::optional<llvm::APSInt> amt =
3099 expr->getArg(Arg: 1)->getIntegerConstantExpr(Ctx: getContext());
3100 assert(amt && "Expected argument to be a constant");
3101 return builder.createShiftRight(
3102 loc, lhs: ops[0], bits: std::min(a: static_cast<uint64_t>(63), b: amt->getZExtValue()));
3103 }
3104 case NEON::BI__builtin_neon_vshrd_n_u64: {
3105 std::optional<llvm::APSInt> amt =
3106 expr->getArg(Arg: 1)->getIntegerConstantExpr(Ctx: getContext());
3107 assert(amt && "Expected argument to be a constant");
3108 uint64_t shiftAmt = amt->getZExtValue();
3109 // Right-shifting an unsigned value by its size yields 0.
3110 if (shiftAmt == 64)
3111 return builder.getConstInt(loc, t: builder.getUInt64Ty(), c: 0);
3112 return builder.createShiftRight(loc, lhs: ops[0], bits: shiftAmt);
3113 }
3114 case NEON::BI__builtin_neon_vsrad_n_s64: {
3115 std::optional<llvm::APSInt> amt =
3116 expr->getArg(Arg: 2)->getIntegerConstantExpr(Ctx: getContext());
3117 assert(amt && "Expected argument to be a constant");
3118 uint64_t shiftAmt =
3119 std::min(a: static_cast<uint64_t>(63), b: amt->getZExtValue());
3120 mlir::Value shifted =
3121 builder.createShiftRight(loc, lhs: ops[1], bits: static_cast<unsigned>(shiftAmt));
3122 return builder.createAdd(loc, lhs: ops[0], rhs: shifted);
3123 }
3124 case NEON::BI__builtin_neon_vsrad_n_u64: {
3125 std::optional<llvm::APSInt> amt =
3126 expr->getArg(Arg: 2)->getIntegerConstantExpr(Ctx: getContext());
3127 assert(amt && "Expected argument to be a constant");
3128 uint64_t shiftAmt = amt->getZExtValue();
3129 // Right-shifting an unsigned value by its size yields 0, so a + 0 = a.
3130 if (shiftAmt == 64)
3131 return ops[0];
3132 mlir::Value shifted =
3133 builder.createShiftRight(loc, lhs: ops[1], bits: static_cast<unsigned>(shiftAmt));
3134 return builder.createAdd(loc, lhs: ops[0], rhs: shifted);
3135 }
3136 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
3137 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
3138 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
3139 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16:
3140 case NEON::BI__builtin_neon_vqdmlals_s32:
3141 case NEON::BI__builtin_neon_vqdmlsls_s32:
3142 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
3143 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
3144 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
3145 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
3146 cgm.errorNYI(expr->getSourceRange(),
3147 std::string("unimplemented AArch64 builtin call: ") +
3148 getContext().BuiltinInfo.getName(ID: builtinID));
3149 return mlir::Value{};
3150 }
3151 case NEON::BI__builtin_neon_vget_lane_bf16:
3152 case NEON::BI__builtin_neon_vduph_lane_bf16:
3153 case NEON::BI__builtin_neon_vduph_lane_f16:
3154 case NEON::BI__builtin_neon_vgetq_lane_bf16:
3155 case NEON::BI__builtin_neon_vduph_laneq_bf16:
3156 case NEON::BI__builtin_neon_vduph_laneq_f16: {
3157 return cir::VecExtractOp::create(builder, location: loc, vec: ops[0], index: ops[1]);
3158 }
3159 case NEON::BI__builtin_neon_vcvt_bf16_f32:
3160 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32:
3161 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32:
3162 cgm.errorNYI(expr->getSourceRange(),
3163 std::string("unimplemented AArch64 builtin call: ") +
3164 getContext().BuiltinInfo.getName(ID: builtinID));
3165 return mlir::Value{};
3166 case NEON::BI__builtin_neon_vcvt_f16_f32: {
3167 auto v4f32 = cir::VectorType::getFixed(elementType: cgm.floatTy, size: 4);
3168 auto v4f16 = cir::VectorType::getFixed(elementType: cgm.fP16Ty, size: 4);
3169 return builder.createFloatingCast(v: builder.createBitcast(src: ops[0], newTy: v4f32),
3170 destType: v4f16);
3171 }
3172 case NEON::BI__builtin_neon_vcvt_f32_f16: {
3173 auto v4f32 = cir::VectorType::getFixed(elementType: cgm.floatTy, size: 4);
3174 auto v4f16 = cir::VectorType::getFixed(elementType: cgm.fP16Ty, size: 4);
3175 return builder.createFloatingCast(v: builder.createBitcast(src: ops[0], newTy: v4f16),
3176 destType: v4f32);
3177 }
3178 case clang::AArch64::BI_InterlockedAdd:
3179 case clang::AArch64::BI_InterlockedAdd_acq:
3180 case clang::AArch64::BI_InterlockedAdd_rel:
3181 case clang::AArch64::BI_InterlockedAdd_nf:
3182 case clang::AArch64::BI_InterlockedAdd64:
3183 case clang::AArch64::BI_InterlockedAdd64_acq:
3184 case clang::AArch64::BI_InterlockedAdd64_rel:
3185 case clang::AArch64::BI_InterlockedAdd64_nf:
3186 cgm.errorNYI(expr->getSourceRange(),
3187 std::string("unimplemented AArch64 builtin call: ") +
3188 getContext().BuiltinInfo.getName(ID: builtinID));
3189 return mlir::Value{};
3190 }
3191
3192 cir::VectorType ty = getNeonType(cgf: this, typeFlags: type);
3193 if (!ty)
3194 return nullptr;
3195
3196 llvm::StringRef intrName;
3197
3198 switch (builtinID) {
3199 default:
3200 return std::nullopt;
3201 case NEON::BI__builtin_neon_vbsl_v:
3202 case NEON::BI__builtin_neon_vbslq_v: {
3203
3204 cir::VectorType bitTy = getIntVecFromVecTy(builder, vecTy: ty);
3205 ops[0] = builder.createBitcast(src: ops[0], newTy: bitTy);
3206 ops[1] = builder.createBitcast(src: ops[1], newTy: bitTy);
3207 ops[2] = builder.createBitcast(src: ops[2], newTy: bitTy);
3208
3209 ops[1] = builder.createAnd(loc, lhs: ops[0], rhs: ops[1]);
3210 ops[2] = builder.createAnd(loc, lhs: builder.createNot(value: ops[0]), rhs: ops[2]);
3211 ops[0] = builder.createOr(loc, lhs: ops[1], rhs: ops[2]);
3212 return builder.createBitcast(src: ops[0], newTy: ty);
3213 }
3214 case NEON::BI__builtin_neon_vfma_lane_v:
3215 case NEON::BI__builtin_neon_vfmaq_lane_v: {
3216 mlir::Value addend = builder.createBitcast(src: ops[0], newTy: ty);
3217 mlir::Value multiplicand = builder.createBitcast(src: ops[1], newTy: ty);
3218 // For vfmaq_lane, the lane source operand is the non-quad vector, so it has
3219 // half as many lanes as the quad result vector. For vfma_lane, it has the
3220 // same shape as the result vector.
3221 cir::VectorType sourceTy = cir::VectorType::getFixed(
3222 elementType: ty.getElementType(), size: builtinID == NEON::BI__builtin_neon_vfmaq_lane_v
3223 ? ty.getFixedSize() / 2
3224 : ty.getFixedSize());
3225 mlir::Value laneSource = builder.createBitcast(src: ops[2], newTy: sourceTy);
3226 laneSource =
3227 emitNeonSplat(builder, loc, v: laneSource, lane: ops[3], resEltCnt: ty.getFixedSize());
3228
3229 llvm::SmallVector<mlir::Value> fmaOps = {multiplicand, laneSource, addend};
3230 return emitNeonCallToOp<cir::FMAOp>(cgm, builder, argTypes: {ty, ty, ty}, args&: fmaOps,
3231 intrinsicName: std::nullopt, funcResTy: ty, loc);
3232 }
3233 case NEON::BI__builtin_neon_vfma_laneq_v: {
3234 // v1f64 fma should be mapped to Neon scalar f64 fma.
3235 if (ty.getElementType() == cgm.doubleTy) {
3236 mlir::Value addend = builder.createBitcast(src: ops[0], newTy: cgm.doubleTy);
3237 mlir::Value multiplicand = builder.createBitcast(src: ops[1], newTy: cgm.doubleTy);
3238 // The laneq source operand is float64x2_t, so the source vector has two
3239 // double lanes.
3240 cir::VectorType sourceTy = cir::VectorType::getFixed(elementType: cgm.doubleTy, size: 2);
3241 mlir::Value laneSource = builder.createBitcast(src: ops[2], newTy: sourceTy);
3242 laneSource = builder.createExtractElement(
3243 loc, vec: laneSource,
3244 idx: static_cast<uint64_t>(getIntValueFromConstOp(val: ops[3])));
3245
3246 llvm::SmallVector<mlir::Value> fmaOps = {multiplicand, laneSource,
3247 addend};
3248 return builder.createBitcast(
3249 src: emitNeonCallToOp<cir::FMAOp>(
3250 cgm, builder, argTypes: {cgm.doubleTy, cgm.doubleTy, cgm.doubleTy}, args&: fmaOps,
3251 intrinsicName: std::nullopt, funcResTy: cgm.doubleTy, loc),
3252 newTy: ty);
3253 }
3254
3255 mlir::Value addend = builder.createBitcast(src: ops[0], newTy: ty);
3256 mlir::Value multiplicand = builder.createBitcast(src: ops[1], newTy: ty);
3257 // The laneq source operand is the quad vector, so it has twice as many
3258 // lanes as the non-quad result vector.
3259 cir::VectorType sourceTy =
3260 cir::VectorType::getFixed(elementType: ty.getElementType(), size: ty.getFixedSize() * 2);
3261 mlir::Value laneSource = builder.createBitcast(src: ops[2], newTy: sourceTy);
3262 laneSource =
3263 emitNeonSplat(builder, loc, v: laneSource, lane: ops[3], resEltCnt: ty.getFixedSize());
3264
3265 llvm::SmallVector<mlir::Value> fmaOps = {laneSource, multiplicand, addend};
3266 return emitNeonCallToOp<cir::FMAOp>(cgm, builder, argTypes: {ty, ty, ty}, args&: fmaOps,
3267 intrinsicName: std::nullopt, funcResTy: ty, loc);
3268 }
3269 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
3270 mlir::Value addend = builder.createBitcast(src: ops[0], newTy: ty);
3271 mlir::Value multiplicand = builder.createBitcast(src: ops[1], newTy: ty);
3272 mlir::Value laneSource = builder.createBitcast(src: ops[2], newTy: ty);
3273 laneSource =
3274 emitNeonSplat(builder, loc, v: laneSource, lane: ops[3], resEltCnt: ty.getFixedSize());
3275
3276 llvm::SmallVector<mlir::Value> fmaOps = {laneSource, multiplicand, addend};
3277 return emitNeonCallToOp<cir::FMAOp>(cgm, builder, argTypes: {ty, ty, ty}, args&: fmaOps,
3278 intrinsicName: std::nullopt, funcResTy: ty, loc);
3279 }
3280 case NEON::BI__builtin_neon_vfmah_lane_f16:
3281 case NEON::BI__builtin_neon_vfmas_lane_f32:
3282 case NEON::BI__builtin_neon_vfmah_laneq_f16:
3283 case NEON::BI__builtin_neon_vfmas_laneq_f32: {
3284 // Scalar lane/laneq forms use one selected element from the lane source.
3285 mlir::Value laneSource = builder.createExtractElement(
3286 loc, vec: ops[2], idx: static_cast<uint64_t>(getIntValueFromConstOp(val: ops[3])));
3287
3288 llvm::SmallVector<mlir::Value> fmaOps = {ops[1], laneSource, ops[0]};
3289 mlir::Type ty = convertType(t: expr->getType());
3290 return emitNeonCallToOp<cir::FMAOp>(cgm, builder, argTypes: {ty, ty, ty}, args&: fmaOps,
3291 intrinsicName: std::nullopt, funcResTy: ty, loc);
3292 }
3293 case NEON::BI__builtin_neon_vfmad_lane_f64:
3294 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
3295 // The lane source operand is float64x1_t for lane forms and float64x2_t
3296 // for laneq forms.
3297 mlir::Value laneSource = builder.createExtractElement(
3298 loc, vec: ops[2], idx: static_cast<uint64_t>(getIntValueFromConstOp(val: ops[3])));
3299
3300 llvm::SmallVector<mlir::Value> fmaOps = {ops[1], laneSource, ops[0]};
3301 return emitNeonCallToOp<cir::FMAOp>(
3302 cgm, builder, argTypes: {cgm.doubleTy, cgm.doubleTy, cgm.doubleTy}, args&: fmaOps,
3303 intrinsicName: std::nullopt, funcResTy: cgm.doubleTy, loc);
3304 }
3305 case NEON::BI__builtin_neon_vmull_v: {
3306 intrName = usgn ? "aarch64.neon.umull" : "aarch64.neon.smull";
3307 if (type.isPoly())
3308 intrName = "aarch64.neon.pmull";
3309 cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
3310 vt: ty, /*isExtended*/ false, isSigned: !usgn);
3311 return emitNeonCall(cgm, builder, argTypes: {argTy, argTy}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3312 }
3313 case NEON::BI__builtin_neon_vmax_v:
3314 case NEON::BI__builtin_neon_vmaxq_v:
3315 intrName = usgn ? "umax" : "smax";
3316 if (cir::isFPOrVectorOfFPType(type: ty))
3317 intrName = "aarch64.neon.fmax";
3318 return emitNeonCall(cgm, builder, argTypes: {ty, ty}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3319 case NEON::BI__builtin_neon_vmaxh_f16: {
3320 auto halfTy = builder.getFp16Ty();
3321 return builder.emitIntrinsicCallOp(loc, str: "aarch64.neon.fmax", resTy: halfTy, op&: ops);
3322 }
3323 case NEON::BI__builtin_neon_vmin_v:
3324 case NEON::BI__builtin_neon_vminq_v:
3325 intrName = usgn ? "umin" : "smin";
3326 if (cir::isFPOrVectorOfFPType(type: ty))
3327 intrName = "aarch64.neon.fmin";
3328 return emitNeonCall(cgm, builder, argTypes: {ty, ty}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3329 case NEON::BI__builtin_neon_vminh_f16: {
3330 auto halfTy = builder.getFp16Ty();
3331 return builder.emitIntrinsicCallOp(loc, str: "aarch64.neon.fmin", resTy: halfTy, op&: ops);
3332 }
3333 case NEON::BI__builtin_neon_vabd_v:
3334 case NEON::BI__builtin_neon_vabdq_v:
3335 intrName = usgn ? "aarch64.neon.uabd" : "aarch64.neon.sabd";
3336 if (cir::isFPOrVectorOfFPType(type: ty))
3337 intrName = "aarch64.neon.fabd";
3338 return emitNeonCall(cgm, builder, argTypes: {ty, ty}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3339 case NEON::BI__builtin_neon_vpadal_v:
3340 case NEON::BI__builtin_neon_vpadalq_v: {
3341 intrName = usgn ? "aarch64.neon.uaddlp" : "aarch64.neon.saddlp";
3342 llvm::SmallVector<mlir::Value> inputs{ops[1]};
3343 mlir::Value pairwiseSum =
3344 emitNeonCall(cgm, builder, argTypes: {getNeonPairwiseWidenInputType(resType: ty, usgn)},
3345 args&: inputs, intrinsicName: intrName, funcResTy: ty, loc);
3346 mlir::Value accumValue = builder.createBitcast(loc, src: ops[0], newTy: ty);
3347 return cir::AddOp::create(builder, location: loc, result: ty, lhs: pairwiseSum, rhs: accumValue);
3348 }
3349 case NEON::BI__builtin_neon_vpmin_v:
3350 case NEON::BI__builtin_neon_vpminq_v:
3351 intrName = usgn ? "aarch64.neon.uminp" : "aarch64.neon.sminp";
3352 if (cir::isFPOrVectorOfFPType(type: ty))
3353 intrName = "aarch64.neon.fminp";
3354 return emitNeonCall(cgm, builder, argTypes: {ty, ty}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3355 case NEON::BI__builtin_neon_vpmax_v:
3356 case NEON::BI__builtin_neon_vpmaxq_v:
3357 intrName = usgn ? "aarch64.neon.umaxp" : "aarch64.neon.smaxp";
3358 if (cir::isFPOrVectorOfFPType(type: ty))
3359 intrName = "aarch64.neon.fmaxp";
3360 return emitNeonCall(cgm, builder, argTypes: {ty, ty}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3361 case NEON::BI__builtin_neon_vminnm_v:
3362 case NEON::BI__builtin_neon_vminnmq_v:
3363 intrName = "aarch64.neon.fminnm";
3364 return emitNeonCall(cgm, builder, argTypes: {ty, ty}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3365 case NEON::BI__builtin_neon_vminnmh_f16: {
3366 auto halfTy = builder.getFp16Ty();
3367 return builder.emitIntrinsicCallOp(loc, str: "aarch64.neon.fminnm", resTy: halfTy, op&: ops);
3368 }
3369 case NEON::BI__builtin_neon_vmaxnm_v:
3370 case NEON::BI__builtin_neon_vmaxnmq_v:
3371 intrName = "aarch64.neon.fmaxnm";
3372 return emitNeonCall(cgm, builder, argTypes: {ty, ty}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3373 case NEON::BI__builtin_neon_vmaxnmh_f16: {
3374 auto halfTy = builder.getFp16Ty();
3375 return builder.emitIntrinsicCallOp(loc, str: "aarch64.neon.fmaxnm", resTy: halfTy, op&: ops);
3376 }
3377 case NEON::BI__builtin_neon_vrecpss_f32:
3378 case NEON::BI__builtin_neon_vrecpsd_f64:
3379 cgm.errorNYI(expr->getSourceRange(),
3380 std::string("unimplemented AArch64 builtin call: ") +
3381 getContext().BuiltinInfo.getName(ID: builtinID));
3382 return mlir::Value{};
3383 case NEON::BI__builtin_neon_vrecpsh_f16: {
3384 auto halfTy = builder.getFp16Ty();
3385 return builder.emitIntrinsicCallOp(loc, str: "aarch64.neon.frecps", resTy: halfTy, op&: ops);
3386 }
3387 case NEON::BI__builtin_neon_vqshrun_n_v: {
3388 cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
3389 vt: ty, /*isExtended=*/true, /*isSigned=*/true);
3390 return emitNeonCall(cgm, builder, argTypes: {argTy, sInt32Ty}, args&: ops,
3391 intrinsicName: "aarch64.neon.sqshrun", funcResTy: ty, loc);
3392 }
3393 case NEON::BI__builtin_neon_vqrshrun_n_v: {
3394 cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
3395 vt: ty, /*isExtended=*/true, /*isSigned=*/true);
3396 return emitNeonCall(cgm, builder, argTypes: {argTy, sInt32Ty}, args&: ops,
3397 intrinsicName: "aarch64.neon.sqrshrun", funcResTy: ty, loc);
3398 }
3399 case NEON::BI__builtin_neon_vqshrn_n_v: {
3400 cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
3401 vt: ty, /*isExtended=*/true, /*isSigned=*/!usgn);
3402 llvm::StringRef intrName =
3403 usgn ? "aarch64.neon.uqshrn" : "aarch64.neon.sqshrn";
3404 return emitNeonCall(cgm, builder, argTypes: {argTy, sInt32Ty}, args&: ops, intrinsicName: intrName, funcResTy: ty,
3405 loc);
3406 }
3407 case NEON::BI__builtin_neon_vrshrn_n_v:
3408 cgm.errorNYI(expr->getSourceRange(),
3409 std::string("unimplemented AArch64 builtin call: ") +
3410 getContext().BuiltinInfo.getName(ID: builtinID));
3411 return mlir::Value{};
3412 case NEON::BI__builtin_neon_vqrshrn_n_v: {
3413 cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
3414 vt: ty, /*isExtended=*/true, /*isSigned=*/!usgn);
3415 llvm::StringRef intrName =
3416 usgn ? "aarch64.neon.uqrshrn" : "aarch64.neon.sqrshrn";
3417 return emitNeonCall(cgm, builder, argTypes: {argTy, sInt32Ty}, args&: ops, intrinsicName: intrName, funcResTy: ty,
3418 loc);
3419 }
3420 case NEON::BI__builtin_neon_vrndah_f16: {
3421 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3422 mlir::Type halfTy = builder.getFp16Ty();
3423 return emitNeonCallToOp<cir::RoundOp>(cgm, builder, argTypes: {halfTy}, args&: ops,
3424 intrinsicName: std::nullopt, funcResTy: halfTy, loc);
3425 }
3426 case NEON::BI__builtin_neon_vrnda_v:
3427 case NEON::BI__builtin_neon_vrndaq_v:
3428 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3429 return emitNeonCallToOp<cir::RoundOp>(cgm, builder, argTypes: {ty}, args&: ops, intrinsicName: std::nullopt,
3430 funcResTy: ty, loc);
3431 case NEON::BI__builtin_neon_vrndih_f16: {
3432 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3433 mlir::Type halfTy = builder.getFp16Ty();
3434 return emitNeonCallToOp<cir::NearbyintOp>(cgm, builder, argTypes: {halfTy}, args&: ops,
3435 intrinsicName: std::nullopt, funcResTy: halfTy, loc);
3436 }
3437 case NEON::BI__builtin_neon_vrndmh_f16: {
3438 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3439 mlir::Type halfTy = builder.getFp16Ty();
3440 return emitNeonCallToOp<cir::FloorOp>(cgm, builder, argTypes: {halfTy}, args&: ops,
3441 intrinsicName: std::nullopt, funcResTy: halfTy, loc);
3442 }
3443 case NEON::BI__builtin_neon_vrndm_v:
3444 case NEON::BI__builtin_neon_vrndmq_v:
3445 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3446 return emitNeonCallToOp<cir::FloorOp>(cgm, builder, argTypes: {ty}, args&: ops, intrinsicName: std::nullopt,
3447 funcResTy: ty, loc);
3448 case NEON::BI__builtin_neon_vrndnh_f16: {
3449 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3450 mlir::Type halfTy = builder.getFp16Ty();
3451 return emitNeonCallToOp<cir::RoundEvenOp>(cgm, builder, argTypes: {halfTy}, args&: ops,
3452 intrinsicName: std::nullopt, funcResTy: halfTy, loc);
3453 }
3454 case NEON::BI__builtin_neon_vrndn_v:
3455 case NEON::BI__builtin_neon_vrndnq_v:
3456 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3457 return emitNeonCallToOp<cir::RoundEvenOp>(cgm, builder, argTypes: {ty}, args&: ops,
3458 intrinsicName: std::nullopt, funcResTy: ty, loc);
3459 case NEON::BI__builtin_neon_vrndns_f32: {
3460 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3461 mlir::Type floatTy = builder.getSingleTy();
3462 return emitNeonCallToOp<cir::RoundEvenOp>(cgm, builder, argTypes: {floatTy}, args&: ops,
3463 intrinsicName: std::nullopt, funcResTy: floatTy, loc);
3464 }
3465 case NEON::BI__builtin_neon_vrndph_f16: {
3466 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3467 mlir::Type halfTy = builder.getFp16Ty();
3468 return emitNeonCallToOp<cir::CeilOp>(cgm, builder, argTypes: {halfTy}, args&: ops,
3469 intrinsicName: std::nullopt, funcResTy: halfTy, loc);
3470 }
3471 case NEON::BI__builtin_neon_vrndp_v:
3472 case NEON::BI__builtin_neon_vrndpq_v:
3473 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3474 return emitNeonCallToOp<cir::CeilOp>(cgm, builder, argTypes: {ty}, args&: ops, intrinsicName: std::nullopt,
3475 funcResTy: ty, loc);
3476 case NEON::BI__builtin_neon_vrndxh_f16: {
3477 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3478 mlir::Type halfTy = builder.getFp16Ty();
3479 return emitNeonCallToOp<cir::RintOp>(cgm, builder, argTypes: {halfTy}, args&: ops,
3480 intrinsicName: std::nullopt, funcResTy: halfTy, loc);
3481 }
3482 case NEON::BI__builtin_neon_vrndx_v:
3483 case NEON::BI__builtin_neon_vrndxq_v:
3484 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3485 return emitNeonCallToOp<cir::RintOp>(cgm, builder, argTypes: {ty}, args&: ops, intrinsicName: std::nullopt,
3486 funcResTy: ty, loc);
3487 case NEON::BI__builtin_neon_vrndh_f16: {
3488 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3489 mlir::Type halfTy = builder.getFp16Ty();
3490 return emitNeonCallToOp<cir::TruncOp>(cgm, builder, argTypes: {halfTy}, args&: ops,
3491 intrinsicName: std::nullopt, funcResTy: halfTy, loc);
3492 }
3493 case NEON::BI__builtin_neon_vrnd_v:
3494 case NEON::BI__builtin_neon_vrndq_v:
3495 assert(!cir::MissingFeatures::emitConstrainedFPCall());
3496 return emitNeonCallToOp<cir::TruncOp>(cgm, builder, argTypes: {ty}, args&: ops, intrinsicName: std::nullopt,
3497 funcResTy: ty, loc);
3498 case NEON::BI__builtin_neon_vcvt_f64_v:
3499 case NEON::BI__builtin_neon_vcvtq_f64_v:
3500 ops[0] = builder.createBitcast(src: ops[0], newTy: ty);
3501 ty = getNeonType(
3502 cgf: this, typeFlags: NeonTypeFlags(NeonTypeFlags::Float64, false, type.isQuad()));
3503 return builder.createCast(loc, kind: cir::CastKind::int_to_float, src: ops[0], newTy: ty);
3504 case NEON::BI__builtin_neon_vcvt_f64_f32: {
3505 auto v2f32 = cir::VectorType::getFixed(elementType: cgm.floatTy, size: 2);
3506 auto v2f64 = cir::VectorType::getFixed(elementType: cgm.doubleTy, size: 2);
3507 return builder.createFloatingCast(v: builder.createBitcast(src: ops[0], newTy: v2f32),
3508 destType: v2f64);
3509 }
3510 case NEON::BI__builtin_neon_vcvt_f32_f64: {
3511 auto v2f32 = cir::VectorType::getFixed(elementType: cgm.floatTy, size: 2);
3512 auto v2f64 = cir::VectorType::getFixed(elementType: cgm.doubleTy, size: 2);
3513 return builder.createFloatingCast(v: builder.createBitcast(src: ops[0], newTy: v2f64),
3514 destType: v2f32);
3515 }
3516 case NEON::BI__builtin_neon_vcvt_s32_v:
3517 case NEON::BI__builtin_neon_vcvt_u32_v:
3518 case NEON::BI__builtin_neon_vcvt_s64_v:
3519 case NEON::BI__builtin_neon_vcvt_u64_v:
3520 case NEON::BI__builtin_neon_vcvt_s16_f16:
3521 case NEON::BI__builtin_neon_vcvt_u16_f16:
3522 case NEON::BI__builtin_neon_vcvtq_s32_v:
3523 case NEON::BI__builtin_neon_vcvtq_u32_v:
3524 case NEON::BI__builtin_neon_vcvtq_s64_v:
3525 case NEON::BI__builtin_neon_vcvtq_u64_v:
3526 case NEON::BI__builtin_neon_vcvtq_s16_f16:
3527 case NEON::BI__builtin_neon_vcvtq_u16_f16:
3528
3529 case NEON::BI__builtin_neon_vcvta_s16_f16:
3530 case NEON::BI__builtin_neon_vcvta_u16_f16:
3531 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
3532 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
3533 cgm.errorNYI(expr->getSourceRange(),
3534 std::string("unimplemented AArch64 builtin call: ") +
3535 getContext().BuiltinInfo.getName(ID: builtinID));
3536 return mlir::Value{};
3537 case NEON::BI__builtin_neon_vcvta_s32_v:
3538 case NEON::BI__builtin_neon_vcvtaq_s32_v:
3539 case NEON::BI__builtin_neon_vcvta_u32_v:
3540 case NEON::BI__builtin_neon_vcvtaq_u32_v:
3541 case NEON::BI__builtin_neon_vcvta_s64_v:
3542 case NEON::BI__builtin_neon_vcvtaq_s64_v:
3543 case NEON::BI__builtin_neon_vcvta_u64_v:
3544 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
3545 auto argTy = getFloatNeonType(cgf&: *this, intTypeFlags: type);
3546 llvm::StringRef intrName =
3547 usgn ? "aarch64.neon.fcvtau" : "aarch64.neon.fcvtas";
3548 return emitNeonCall(cgm, builder, argTypes: {argTy}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3549 }
3550
3551 case NEON::BI__builtin_neon_vcvtm_s16_f16:
3552 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
3553 case NEON::BI__builtin_neon_vcvtm_u16_f16:
3554 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
3555 cgm.errorNYI(expr->getSourceRange(),
3556 std::string("unimplemented AArch64 builtin call: ") +
3557 getContext().BuiltinInfo.getName(ID: builtinID));
3558 return mlir::Value{};
3559 case NEON::BI__builtin_neon_vcvtm_s32_v:
3560 case NEON::BI__builtin_neon_vcvtmq_s32_v:
3561 case NEON::BI__builtin_neon_vcvtm_u32_v:
3562 case NEON::BI__builtin_neon_vcvtmq_u32_v:
3563 case NEON::BI__builtin_neon_vcvtm_s64_v:
3564 case NEON::BI__builtin_neon_vcvtmq_s64_v:
3565 case NEON::BI__builtin_neon_vcvtm_u64_v:
3566 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
3567 auto argTy = getFloatNeonType(cgf&: *this, intTypeFlags: type);
3568 llvm::StringRef intrName =
3569 usgn ? "aarch64.neon.fcvtmu" : "aarch64.neon.fcvtms";
3570 return emitNeonCall(cgm, builder, argTypes: {argTy}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3571 }
3572
3573 case NEON::BI__builtin_neon_vcvtn_s16_f16:
3574 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
3575 case NEON::BI__builtin_neon_vcvtn_u16_f16:
3576 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
3577 cgm.errorNYI(expr->getSourceRange(),
3578 std::string("unimplemented AArch64 builtin call: ") +
3579 getContext().BuiltinInfo.getName(ID: builtinID));
3580 return mlir::Value{};
3581
3582 case NEON::BI__builtin_neon_vcvtn_s32_v:
3583 case NEON::BI__builtin_neon_vcvtnq_s32_v:
3584 case NEON::BI__builtin_neon_vcvtn_u32_v:
3585 case NEON::BI__builtin_neon_vcvtnq_u32_v:
3586 case NEON::BI__builtin_neon_vcvtn_s64_v:
3587 case NEON::BI__builtin_neon_vcvtnq_s64_v:
3588 case NEON::BI__builtin_neon_vcvtn_u64_v:
3589 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
3590 auto argTy = getFloatNeonType(cgf&: *this, intTypeFlags: type);
3591 llvm::StringRef intrName =
3592 usgn ? "aarch64.neon.fcvtnu" : "aarch64.neon.fcvtns";
3593 return emitNeonCall(cgm, builder, argTypes: {argTy}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3594 }
3595
3596 case NEON::BI__builtin_neon_vcvtp_s16_f16:
3597 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
3598 case NEON::BI__builtin_neon_vcvtp_u16_f16:
3599 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
3600 cgm.errorNYI(expr->getSourceRange(),
3601 std::string("unimplemented AArch64 builtin call: ") +
3602 getContext().BuiltinInfo.getName(ID: builtinID));
3603 return mlir::Value{};
3604 case NEON::BI__builtin_neon_vcvtp_s32_v:
3605 case NEON::BI__builtin_neon_vcvtpq_s32_v:
3606 case NEON::BI__builtin_neon_vcvtp_u32_v:
3607 case NEON::BI__builtin_neon_vcvtpq_u32_v:
3608 case NEON::BI__builtin_neon_vcvtp_s64_v:
3609 case NEON::BI__builtin_neon_vcvtpq_s64_v:
3610 case NEON::BI__builtin_neon_vcvtp_u64_v:
3611 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
3612 auto argTy = getFloatNeonType(cgf&: *this, intTypeFlags: type);
3613 llvm::StringRef intrName =
3614 usgn ? "aarch64.neon.fcvtpu" : "aarch64.neon.fcvtps";
3615 return emitNeonCall(cgm, builder, argTypes: {argTy}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3616 }
3617
3618 case NEON::BI__builtin_neon_vmulx_v:
3619 case NEON::BI__builtin_neon_vmulxq_v:
3620 case NEON::BI__builtin_neon_vmulxh_lane_f16:
3621 case NEON::BI__builtin_neon_vmulxh_laneq_f16:
3622 case NEON::BI__builtin_neon_vmul_lane_v:
3623 case NEON::BI__builtin_neon_vmul_laneq_v:
3624 case NEON::BI__builtin_neon_vpmaxnm_v:
3625 case NEON::BI__builtin_neon_vpmaxnmq_v:
3626 cgm.errorNYI(expr->getSourceRange(),
3627 std::string("unimplemented AArch64 builtin call: ") +
3628 getContext().BuiltinInfo.getName(ID: builtinID));
3629 return mlir::Value{};
3630 case NEON::BI__builtin_neon_vpminnm_v:
3631 case NEON::BI__builtin_neon_vpminnmq_v:
3632 intrName = "aarch64.neon.fminnmp";
3633 return emitNeonCall(cgm, builder, argTypes: {ty, ty}, args&: ops, intrinsicName: intrName, funcResTy: ty, loc);
3634 case NEON::BI__builtin_neon_vsqrth_f16: {
3635 mlir::Type halfTy = builder.getFp16Ty();
3636 return emitNeonCallToOp<cir::SqrtOp>(cgm, builder, argTypes: {halfTy}, args&: ops,
3637 intrinsicName: std::nullopt, funcResTy: halfTy, loc);
3638 }
3639 case NEON::BI__builtin_neon_vsqrt_v:
3640 case NEON::BI__builtin_neon_vsqrtq_v:
3641 return emitNeonCallToOp<cir::SqrtOp>(cgm, builder, argTypes: {ty}, args&: ops, intrinsicName: std::nullopt,
3642 funcResTy: ty, loc);
3643 case NEON::BI__builtin_neon_vrbit_v:
3644 case NEON::BI__builtin_neon_vrbitq_v:
3645 case NEON::BI__builtin_neon_vmaxv_f16:
3646 case NEON::BI__builtin_neon_vmaxvq_f16:
3647 case NEON::BI__builtin_neon_vminv_f16:
3648 case NEON::BI__builtin_neon_vminvq_f16:
3649 case NEON::BI__builtin_neon_vmaxnmv_f16:
3650 case NEON::BI__builtin_neon_vmaxnmvq_f16:
3651 case NEON::BI__builtin_neon_vminnmv_f16:
3652 case NEON::BI__builtin_neon_vminnmvq_f16:
3653 case NEON::BI__builtin_neon_vmul_n_f64:
3654 cgm.errorNYI(expr->getSourceRange(),
3655 std::string("unimplemented AArch64 builtin call: ") +
3656 getContext().BuiltinInfo.getName(ID: builtinID));
3657 return mlir::Value{};
3658 case NEON::BI__builtin_neon_vaddlv_u8:
3659 case NEON::BI__builtin_neon_vaddlvq_u8:
3660 case NEON::BI__builtin_neon_vaddlv_u16:
3661 case NEON::BI__builtin_neon_vaddlvq_u16:
3662 case NEON::BI__builtin_neon_vaddlv_s8:
3663 case NEON::BI__builtin_neon_vaddlvq_s8:
3664 case NEON::BI__builtin_neon_vaddlv_s16:
3665 case NEON::BI__builtin_neon_vaddlvq_s16: {
3666 mlir::Type argTy = convertType(t: expr->getArg(Arg: 0)->getType());
3667 mlir::Type userRetTy = convertType(t: expr->getType());
3668 auto eltTy = mlir::cast<cir::IntType>(
3669 Val: mlir::cast<cir::VectorType>(Val&: argTy).getElementType());
3670 bool isUnsigned = !eltTy.isSigned();
3671 // These builtins only use 8 and 16-bit element vectors; the intrinsic
3672 // always produces i32. The C result is i32 for 16-bit elements, but i16
3673 // for 8-bit elements, so we emit at i32 and narrow only in that case.
3674 bool needsTrunc = eltTy.getWidth() == 8;
3675 intrName = isUnsigned ? "aarch64.neon.uaddlv" : "aarch64.neon.saddlv";
3676 mlir::Type intrRetTy = userRetTy;
3677 if (needsTrunc)
3678 intrRetTy = isUnsigned ? builder.getUInt32Ty() : builder.getSInt32Ty();
3679 mlir::Value result =
3680 emitNeonCall(cgm, builder, argTypes: {argTy}, args&: ops, intrinsicName: intrName, funcResTy: intrRetTy, loc);
3681 if (needsTrunc)
3682 result = builder.createIntCast(src: result, newTy: userRetTy);
3683 return result;
3684 }
3685 case NEON::BI__builtin_neon_vsri_n_v:
3686 case NEON::BI__builtin_neon_vsriq_n_v: {
3687 llvm::SmallVector<mlir::Value, 4> vsriArgs = {
3688 ops[0], ops[1], builder.createIntCast(src: ops[2], newTy: builder.getUInt32Ty())};
3689 return emitNeonCall(cgm, builder, argTypes: {ty, ty, builder.getUInt32Ty()}, args&: vsriArgs,
3690 intrinsicName: "aarch64.neon.vsri", funcResTy: ty, loc);
3691 }
3692 case NEON::BI__builtin_neon_vsli_n_v:
3693 case NEON::BI__builtin_neon_vsliq_n_v: {
3694
3695 intrName = "aarch64.neon.vsli";
3696
3697 llvm::SmallVector<mlir::Type> argTypes = {ty, ty, ops[2].getType()};
3698
3699 return emitNeonCall(cgm, builder, argTypes, args&: ops, intrinsicName: intrName, funcResTy: ty, loc,
3700 /*isConstrainedFPIntrinsic=*/false,
3701 /*shift=*/0, /*rightshift=*/false);
3702 }
3703 case NEON::BI__builtin_neon_vsra_n_v:
3704 case NEON::BI__builtin_neon_vsraq_n_v: {
3705 ops[0] = builder.createBitcast(src: ops[0], newTy: ty);
3706 ops[1] = emitNeonRShiftImm(cgf&: *this, shiftVec: ops[1], shiftVal: ops[2], vecTy: ty, usgn, loc);
3707 return builder.createAdd(loc, lhs: ops[0], rhs: ops[1]);
3708 }
3709 case NEON::BI__builtin_neon_vrsra_n_v:
3710 case NEON::BI__builtin_neon_vrsraq_n_v: {
3711 intrName = usgn ? "aarch64.neon.urshl" : "aarch64.neon.srshl";
3712 // The llvm intrinsic is expecting negative shift amount for right shift.
3713 // Thus we have to make shift amount vec type to be signed.
3714 cir::VectorType shiftAmtVecTy =
3715 usgn ? getSignChangedVectorType(builder, vecTy: ty) : ty;
3716 llvm::SmallVector<mlir::Value, 2> tmpOps = {ops[1], ops[2]};
3717 mlir::Value tmp = emitNeonCall(cgm, builder, argTypes: {ty, shiftAmtVecTy}, args&: tmpOps,
3718 intrinsicName: intrName, funcResTy: ty, loc,
3719 /*isConstrainedFPIntrinsic=*/false,
3720 /*shift=*/1, /*rightshift=*/true);
3721 ops[0] = builder.createBitcast(src: ops[0], newTy: ty);
3722 return builder.createAdd(loc, lhs: ops[0], rhs: tmp);
3723 }
3724 case NEON::BI__builtin_neon_vld1_v:
3725 case NEON::BI__builtin_neon_vld1q_v:
3726 return builder.createLoad(loc, addr: ptrOp0.withElementType(builder, ElemTy: ty));
3727 case NEON::BI__builtin_neon_vst1_v:
3728 case NEON::BI__builtin_neon_vst1q_v: {
3729 ops[1] = builder.createBitcast(src: ops[1], newTy: ty);
3730 builder.createStore(loc, val: ops[1], dst: ptrOp0.withElementType(builder, ElemTy: ty));
3731 return nullptr;
3732 }
3733 case NEON::BI__builtin_neon_vld1_lane_v:
3734 case NEON::BI__builtin_neon_vld1q_lane_v: {
3735 ops[1] = builder.createBitcast(src: ops[1], newTy: ty);
3736 mlir::Value scalar = builder.createLoad(
3737 loc, addr: ptrOp0.withElementType(builder, ElemTy: ty.getElementType()));
3738 return cir::VecInsertOp::create(builder, location: loc, vec: ops[1], value: scalar, index: ops[2]);
3739 }
3740 case NEON::BI__builtin_neon_vld1_dup_v:
3741 case NEON::BI__builtin_neon_vld1q_dup_v: {
3742 mlir::Value scalar = builder.createLoad(
3743 loc, addr: ptrOp0.withElementType(builder, ElemTy: ty.getElementType()));
3744 return cir::VecSplatOp::create(builder, location: loc, result: ty, value: scalar);
3745 }
3746 case NEON::BI__builtin_neon_vldap1_lane_s64:
3747 case NEON::BI__builtin_neon_vldap1q_lane_s64:
3748 cgm.errorNYI(expr->getSourceRange(),
3749 std::string("unimplemented AArch64 builtin call: ") +
3750 getContext().BuiltinInfo.getName(ID: builtinID));
3751 return mlir::Value{};
3752 case NEON::BI__builtin_neon_vst1_lane_v:
3753 case NEON::BI__builtin_neon_vst1q_lane_v: {
3754 ops[1] = builder.createBitcast(src: ops[1], newTy: ty);
3755 mlir::Value scalar = builder.createExtractElement(
3756 loc, vec: ops[1], idx: static_cast<uint64_t>(getIntValueFromConstOp(val: ops[2])));
3757 builder.createStore(loc, val: scalar,
3758 dst: ptrOp0.withElementType(builder, ElemTy: scalar.getType()));
3759 return nullptr;
3760 }
3761 case NEON::BI__builtin_neon_vstl1_lane_s64:
3762 case NEON::BI__builtin_neon_vstl1q_lane_s64:
3763 cgm.errorNYI(expr->getSourceRange(),
3764 std::string("unimplemented AArch64 builtin call: ") +
3765 getContext().BuiltinInfo.getName(ID: builtinID));
3766 return mlir::Value{};
3767 case NEON::BI__builtin_neon_vld2_v:
3768 case NEON::BI__builtin_neon_vld2q_v:
3769 return emitNeonMultiVecLoad(cgf&: *this, ops, llvmIntrinsic: llvm::Intrinsic::aarch64_neon_ld2,
3770 ty, numVecs: 2, loc);
3771 case NEON::BI__builtin_neon_vld3_v:
3772 case NEON::BI__builtin_neon_vld3q_v:
3773 return emitNeonMultiVecLoad(cgf&: *this, ops, llvmIntrinsic: llvm::Intrinsic::aarch64_neon_ld3,
3774 ty, numVecs: 3, loc);
3775 case NEON::BI__builtin_neon_vld4_v:
3776 case NEON::BI__builtin_neon_vld4q_v:
3777 return emitNeonMultiVecLoad(cgf&: *this, ops, llvmIntrinsic: llvm::Intrinsic::aarch64_neon_ld4,
3778 ty, numVecs: 4, loc);
3779 case NEON::BI__builtin_neon_vld2_dup_v:
3780 case NEON::BI__builtin_neon_vld2q_dup_v:
3781 return emitNeonMultiVecLoad(cgf&: *this, ops, llvmIntrinsic: llvm::Intrinsic::aarch64_neon_ld2r,
3782 ty, numVecs: 2, loc);
3783 case NEON::BI__builtin_neon_vld3_dup_v:
3784 case NEON::BI__builtin_neon_vld3q_dup_v:
3785 return emitNeonMultiVecLoad(cgf&: *this, ops, llvmIntrinsic: llvm::Intrinsic::aarch64_neon_ld3r,
3786 ty, numVecs: 3, loc);
3787 case NEON::BI__builtin_neon_vld4_dup_v:
3788 case NEON::BI__builtin_neon_vld4q_dup_v:
3789 return emitNeonMultiVecLoad(cgf&: *this, ops, llvmIntrinsic: llvm::Intrinsic::aarch64_neon_ld4r,
3790 ty, numVecs: 4, loc);
3791 case NEON::BI__builtin_neon_vld2_lane_v:
3792 case NEON::BI__builtin_neon_vld2q_lane_v:
3793 return emitNeonMultiVecLoadLane(
3794 cgf&: *this, ops, llvmIntrinsic: llvm::Intrinsic::aarch64_neon_ld2lane, ty, numVecs: 2, loc);
3795 case NEON::BI__builtin_neon_vld3_lane_v:
3796 case NEON::BI__builtin_neon_vld3q_lane_v:
3797 return emitNeonMultiVecLoadLane(
3798 cgf&: *this, ops, llvmIntrinsic: llvm::Intrinsic::aarch64_neon_ld3lane, ty, numVecs: 3, loc);
3799 case NEON::BI__builtin_neon_vld4_lane_v:
3800 case NEON::BI__builtin_neon_vld4q_lane_v:
3801 return emitNeonMultiVecLoadLane(
3802 cgf&: *this, ops, llvmIntrinsic: llvm::Intrinsic::aarch64_neon_ld4lane, ty, numVecs: 4, loc);
3803 case NEON::BI__builtin_neon_vst2_v:
3804 case NEON::BI__builtin_neon_vst2q_v: {
3805 // The builtin call has the pointer first, but the AArch64 st2 intrinsic
3806 // takes the vector operands first and the pointer last.
3807 std::rotate(first: ops.begin(), middle: ops.begin() + 1, last: ops.end());
3808 llvm::SmallVector<mlir::Type> argTypes = {ty, ty, builder.getVoidPtrTy()};
3809 return emitNeonCall(cgm, builder, argTypes, args&: ops, intrinsicName: "aarch64.neon.st2",
3810 funcResTy: cgm.voidTy, loc);
3811 }
3812 case NEON::BI__builtin_neon_vst2_lane_v:
3813 case NEON::BI__builtin_neon_vst2q_lane_v: {
3814 std::rotate(first: ops.begin(), middle: ops.begin() + 1, last: ops.end());
3815 ops[2] = builder.createIntCast(src: ops[2], newTy: sInt64Ty);
3816 llvm::SmallVector<mlir::Type> argTypes = {ty, ty, sInt64Ty,
3817 builder.getVoidPtrTy()};
3818 return emitNeonCall(cgm, builder, argTypes, args&: ops, intrinsicName: "aarch64.neon.st2lane",
3819 funcResTy: cgm.voidTy, loc);
3820 }
3821 case NEON::BI__builtin_neon_vst3_v:
3822 case NEON::BI__builtin_neon_vst3q_v: {
3823 std::rotate(first: ops.begin(), middle: ops.begin() + 1, last: ops.end());
3824 llvm::SmallVector<mlir::Type> argTypes = {ty, ty, ty,
3825 builder.getVoidPtrTy()};
3826 return emitNeonCall(cgm, builder, argTypes, args&: ops, intrinsicName: "aarch64.neon.st3",
3827 funcResTy: cgm.voidTy, loc);
3828 }
3829 case NEON::BI__builtin_neon_vst3_lane_v:
3830 case NEON::BI__builtin_neon_vst3q_lane_v: {
3831 std::rotate(first: ops.begin(), middle: ops.begin() + 1, last: ops.end());
3832 ops[3] = builder.createIntCast(src: ops[3], newTy: sInt64Ty);
3833 llvm::SmallVector<mlir::Type> argTypes = {ty, ty, ty, sInt64Ty,
3834 builder.getVoidPtrTy()};
3835 return emitNeonCall(cgm, builder, argTypes, args&: ops, intrinsicName: "aarch64.neon.st3lane",
3836 funcResTy: cgm.voidTy, loc);
3837 }
3838 case NEON::BI__builtin_neon_vst4_v:
3839 case NEON::BI__builtin_neon_vst4q_v: {
3840 std::rotate(first: ops.begin(), middle: ops.begin() + 1, last: ops.end());
3841 llvm::SmallVector<mlir::Type> argTypes = {ty, ty, ty, ty,
3842 builder.getVoidPtrTy()};
3843 return emitNeonCall(cgm, builder, argTypes, args&: ops, intrinsicName: "aarch64.neon.st4",
3844 funcResTy: cgm.voidTy, loc);
3845 }
3846 case NEON::BI__builtin_neon_vst4_lane_v:
3847 case NEON::BI__builtin_neon_vst4q_lane_v: {
3848 std::rotate(first: ops.begin(), middle: ops.begin() + 1, last: ops.end());
3849 ops[4] = builder.createIntCast(src: ops[4], newTy: sInt64Ty);
3850 llvm::SmallVector<mlir::Type> argTypes = {
3851 ty, ty, ty, ty, sInt64Ty, builder.getVoidPtrTy()};
3852 return emitNeonCall(cgm, builder, argTypes, args&: ops, intrinsicName: "aarch64.neon.st4lane",
3853 funcResTy: cgm.voidTy, loc);
3854 }
3855 case NEON::BI__builtin_neon_vtrn_v:
3856 case NEON::BI__builtin_neon_vtrnq_v: {
3857 ops[1] = builder.createBitcast(src: ops[1], newTy: ty);
3858 ops[2] = builder.createBitcast(src: ops[2], newTy: ty);
3859 // Adding a bitcast here as Ops[0] might be a void pointer.
3860 mlir::Value baseAddr =
3861 builder.createBitcast(src: ops[0], newTy: builder.getPointerTo(ty));
3862 mlir::Value sv;
3863
3864 for (unsigned vi = 0; vi != 2; ++vi) {
3865 llvm::SmallVector<int64_t, 16> indices;
3866 for (unsigned i = 0, e = ty.getFixedSize(); i != e; i += 2) {
3867 indices.push_back(Elt: i + vi);
3868 indices.push_back(Elt: i + e + vi);
3869 }
3870 cir::ConstantOp idx = builder.getConstInt(loc, t: builder.getSInt32Ty(), c: vi);
3871 mlir::Value addr = builder.createPtrStride(loc, base: baseAddr, stride: idx);
3872 sv = builder.createVecShuffle(loc, vec1: ops[1], vec2: ops[2], mask: indices);
3873 (void)builder.CIRBaseBuilderTy::createStore(loc, val: sv, dst: addr);
3874 }
3875 return sv;
3876 }
3877 case NEON::BI__builtin_neon_vuzp_v:
3878 case NEON::BI__builtin_neon_vuzpq_v: {
3879 ops[1] = builder.createBitcast(src: ops[1], newTy: ty);
3880 ops[2] = builder.createBitcast(src: ops[2], newTy: ty);
3881 // Adding a bitcast here as Ops[0] might be a void pointer.
3882 mlir::Value baseAddr =
3883 builder.createBitcast(src: ops[0], newTy: builder.getPointerTo(ty));
3884 mlir::Value sv;
3885 for (unsigned vi = 0; vi != 2; ++vi) {
3886 llvm::SmallVector<int64_t, 16> indices;
3887 for (unsigned i = 0, e = ty.getFixedSize(); i != e; ++i) {
3888 indices.push_back(Elt: 2 * i + vi);
3889 }
3890 cir::ConstantOp idx = builder.getConstInt(loc, t: builder.getSInt32Ty(), c: vi);
3891 mlir::Value addr = builder.createPtrStride(loc, base: baseAddr, stride: idx);
3892 sv = builder.createVecShuffle(loc, vec1: ops[1], vec2: ops[2], mask: indices);
3893 (void)builder.CIRBaseBuilderTy::createStore(loc, val: sv, dst: addr);
3894 }
3895 return sv;
3896 }
3897 case NEON::BI__builtin_neon_vzip_v:
3898 case NEON::BI__builtin_neon_vzipq_v: {
3899 ops[1] = builder.createBitcast(src: ops[1], newTy: ty);
3900 ops[2] = builder.createBitcast(src: ops[2], newTy: ty);
3901 // Adding a bitcast here as Ops[0] might be a void pointer.
3902 mlir::Value baseAddr =
3903 builder.createBitcast(src: ops[0], newTy: builder.getPointerTo(ty));
3904 mlir::Value sv;
3905 for (unsigned vi = 0; vi != 2; ++vi) {
3906 llvm::SmallVector<int64_t, 16> indices;
3907 for (unsigned i = 0, e = ty.getFixedSize(); i != e; i += 2) {
3908 indices.push_back(Elt: (i + vi * e) >> 1);
3909 indices.push_back(Elt: ((i + vi * e) >> 1) + e);
3910 }
3911 cir::ConstantOp idx = builder.getConstInt(loc, t: builder.getSInt32Ty(), c: vi);
3912 mlir::Value addr = builder.createPtrStride(loc, base: baseAddr, stride: idx);
3913 sv = builder.createVecShuffle(loc, vec1: ops[1], vec2: ops[2], mask: indices);
3914 (void)builder.CIRBaseBuilderTy::createStore(loc, val: sv, dst: addr);
3915 }
3916 return sv;
3917 }
3918 case NEON::BI__builtin_neon_vqtbl1q_v:
3919 case NEON::BI__builtin_neon_vqtbl2q_v:
3920 case NEON::BI__builtin_neon_vqtbl3q_v:
3921 case NEON::BI__builtin_neon_vqtbl4q_v:
3922 case NEON::BI__builtin_neon_vqtbx1q_v:
3923 case NEON::BI__builtin_neon_vqtbx2q_v:
3924 case NEON::BI__builtin_neon_vqtbx3q_v:
3925 case NEON::BI__builtin_neon_vqtbx4q_v:
3926 cgm.errorNYI(expr->getSourceRange(),
3927 std::string("unimplemented AArch64 builtin call: ") +
3928 getContext().BuiltinInfo.getName(ID: builtinID));
3929 return mlir::Value{};
3930 case NEON::BI__builtin_neon_vsqadd_v:
3931 case NEON::BI__builtin_neon_vsqaddq_v:
3932 return emitNeonCall(cgm, builder, argTypes: {ty, ty}, args&: ops, intrinsicName: "aarch64.neon.usqadd", funcResTy: ty,
3933 loc);
3934 case NEON::BI__builtin_neon_vuqadd_v:
3935 case NEON::BI__builtin_neon_vuqaddq_v:
3936 return emitNeonCall(cgm, builder, argTypes: {ty, ty}, args&: ops, intrinsicName: "aarch64.neon.suqadd", funcResTy: ty,
3937 loc);
3938 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
3939 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
3940 case NEON::BI__builtin_neon_vluti2_laneq_f16:
3941 case NEON::BI__builtin_neon_vluti2_laneq_p16:
3942 case NEON::BI__builtin_neon_vluti2_laneq_p8:
3943 case NEON::BI__builtin_neon_vluti2_laneq_s16:
3944 case NEON::BI__builtin_neon_vluti2_laneq_s8:
3945 case NEON::BI__builtin_neon_vluti2_laneq_u16:
3946 case NEON::BI__builtin_neon_vluti2_laneq_u8:
3947 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
3948 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
3949 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
3950 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
3951 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
3952 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
3953 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
3954 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
3955 case NEON::BI__builtin_neon_vluti2q_laneq_u8:
3956 case NEON::BI__builtin_neon_vluti2_lane_mf8:
3957 case NEON::BI__builtin_neon_vluti2_lane_bf16:
3958 case NEON::BI__builtin_neon_vluti2_lane_f16:
3959 case NEON::BI__builtin_neon_vluti2_lane_p16:
3960 case NEON::BI__builtin_neon_vluti2_lane_p8:
3961 case NEON::BI__builtin_neon_vluti2_lane_s16:
3962 case NEON::BI__builtin_neon_vluti2_lane_s8:
3963 case NEON::BI__builtin_neon_vluti2_lane_u16:
3964 case NEON::BI__builtin_neon_vluti2_lane_u8:
3965 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
3966 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
3967 case NEON::BI__builtin_neon_vluti2q_lane_f16:
3968 case NEON::BI__builtin_neon_vluti2q_lane_p16:
3969 case NEON::BI__builtin_neon_vluti2q_lane_p8:
3970 case NEON::BI__builtin_neon_vluti2q_lane_s16:
3971 case NEON::BI__builtin_neon_vluti2q_lane_s8:
3972 case NEON::BI__builtin_neon_vluti2q_lane_u16:
3973 case NEON::BI__builtin_neon_vluti2q_lane_u8:
3974 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
3975 case NEON::BI__builtin_neon_vluti4q_lane_p8:
3976 case NEON::BI__builtin_neon_vluti4q_lane_s8:
3977 case NEON::BI__builtin_neon_vluti4q_lane_u8:
3978 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
3979 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
3980 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
3981 case NEON::BI__builtin_neon_vluti4q_laneq_u8:
3982 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
3983 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
3984 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
3985 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
3986 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2:
3987 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
3988 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
3989 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
3990 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
3991 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2:
3992 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
3993 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
3994 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
3995 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
3996 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
3997 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
3998 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
3999 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
4000 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
4001 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
4002 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
4003 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
4004 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
4005 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
4006 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
4007 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
4008 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
4009 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm:
4010 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
4011 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
4012 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
4013 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
4014 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
4015 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
4016 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
4017 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
4018 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
4019 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
4020 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
4021 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
4022 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
4023 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
4024 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
4025 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
4026 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
4027 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
4028 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
4029 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
4030 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
4031 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
4032 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
4033 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
4034 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
4035 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
4036 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
4037 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
4038 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
4039 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
4040 case NEON::BI__builtin_neon_vamin_f16:
4041 case NEON::BI__builtin_neon_vaminq_f16:
4042 case NEON::BI__builtin_neon_vamin_f32:
4043 case NEON::BI__builtin_neon_vaminq_f32:
4044 case NEON::BI__builtin_neon_vaminq_f64:
4045 case NEON::BI__builtin_neon_vamax_f16:
4046 case NEON::BI__builtin_neon_vamaxq_f16:
4047 case NEON::BI__builtin_neon_vamax_f32:
4048 case NEON::BI__builtin_neon_vamaxq_f32:
4049 case NEON::BI__builtin_neon_vamaxq_f64:
4050 case NEON::BI__builtin_neon_vscale_f16:
4051 case NEON::BI__builtin_neon_vscaleq_f16:
4052 case NEON::BI__builtin_neon_vscale_f32:
4053 case NEON::BI__builtin_neon_vscaleq_f32:
4054 case NEON::BI__builtin_neon_vscaleq_f64:
4055 cgm.errorNYI(expr->getSourceRange(),
4056 std::string("unimplemented AArch64 builtin call: ") +
4057 getContext().BuiltinInfo.getName(ID: builtinID));
4058 return mlir::Value{};
4059 }
4060
4061 // Unreachable: All cases in the switch above return.
4062}
4063