1//===---------- ARM.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfo.h"
14#include "CGBuiltin.h"
15#include "CGDebugInfo.h"
16#include "TargetInfo.h"
17#include "clang/Basic/AArch64CodeGenUtils.h"
18#include "clang/Basic/TargetBuiltins.h"
19#include "clang/CodeGenUtils/TargetUtils.h"
20#include "llvm/IR/InlineAsm.h"
21#include "llvm/IR/IntrinsicsAArch64.h"
22#include "llvm/IR/IntrinsicsARM.h"
23#include "llvm/IR/IntrinsicsBPF.h"
24#include "llvm/Support/AArch64MemoryHints.h"
25#include "llvm/TargetParser/AArch64TargetParser.h"
26
27#include <numeric>
28
29using namespace clang;
30using namespace CodeGen;
31using namespace llvm;
32using namespace clang::aarch64;
33
34static std::optional<CodeGenFunction::MSVCIntrin>
35translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
36 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
37 switch (BuiltinID) {
38 default:
39 return std::nullopt;
40 case clang::AArch64::BI_BitScanForward:
41 case clang::AArch64::BI_BitScanForward64:
42 return MSVCIntrin::_BitScanForward;
43 case clang::AArch64::BI_BitScanReverse:
44 case clang::AArch64::BI_BitScanReverse64:
45 return MSVCIntrin::_BitScanReverse;
46 case clang::AArch64::BI_InterlockedAnd64:
47 return MSVCIntrin::_InterlockedAnd;
48 case clang::AArch64::BI_InterlockedExchange64:
49 return MSVCIntrin::_InterlockedExchange;
50 case clang::AArch64::BI_InterlockedExchangeAdd64:
51 return MSVCIntrin::_InterlockedExchangeAdd;
52 case clang::AArch64::BI_InterlockedExchangeSub64:
53 return MSVCIntrin::_InterlockedExchangeSub;
54 case clang::AArch64::BI_InterlockedOr64:
55 return MSVCIntrin::_InterlockedOr;
56 case clang::AArch64::BI_InterlockedXor64:
57 return MSVCIntrin::_InterlockedXor;
58 case clang::AArch64::BI_InterlockedDecrement64:
59 return MSVCIntrin::_InterlockedDecrement;
60 case clang::AArch64::BI_InterlockedIncrement64:
61 return MSVCIntrin::_InterlockedIncrement;
62 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
63 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
64 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
65 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
66 return MSVCIntrin::_InterlockedExchangeAdd_acq;
67 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
68 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
69 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
70 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
71 return MSVCIntrin::_InterlockedExchangeAdd_rel;
72 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
73 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
74 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
75 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
76 return MSVCIntrin::_InterlockedExchangeAdd_nf;
77 case clang::AArch64::BI_InterlockedExchange8_acq:
78 case clang::AArch64::BI_InterlockedExchange16_acq:
79 case clang::AArch64::BI_InterlockedExchange_acq:
80 case clang::AArch64::BI_InterlockedExchange64_acq:
81 case clang::AArch64::BI_InterlockedExchangePointer_acq:
82 return MSVCIntrin::_InterlockedExchange_acq;
83 case clang::AArch64::BI_InterlockedExchange8_rel:
84 case clang::AArch64::BI_InterlockedExchange16_rel:
85 case clang::AArch64::BI_InterlockedExchange_rel:
86 case clang::AArch64::BI_InterlockedExchange64_rel:
87 case clang::AArch64::BI_InterlockedExchangePointer_rel:
88 return MSVCIntrin::_InterlockedExchange_rel;
89 case clang::AArch64::BI_InterlockedExchange8_nf:
90 case clang::AArch64::BI_InterlockedExchange16_nf:
91 case clang::AArch64::BI_InterlockedExchange_nf:
92 case clang::AArch64::BI_InterlockedExchange64_nf:
93 case clang::AArch64::BI_InterlockedExchangePointer_nf:
94 return MSVCIntrin::_InterlockedExchange_nf;
95 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
96 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
97 case clang::AArch64::BI_InterlockedCompareExchange_acq:
98 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
99 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
100 return MSVCIntrin::_InterlockedCompareExchange_acq;
101 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
102 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
103 case clang::AArch64::BI_InterlockedCompareExchange_rel:
104 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
105 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
106 return MSVCIntrin::_InterlockedCompareExchange_rel;
107 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
108 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
109 case clang::AArch64::BI_InterlockedCompareExchange_nf:
110 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
111 return MSVCIntrin::_InterlockedCompareExchange_nf;
112 case clang::AArch64::BI_InterlockedCompareExchange128:
113 return MSVCIntrin::_InterlockedCompareExchange128;
114 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
115 return MSVCIntrin::_InterlockedCompareExchange128_acq;
116 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
117 return MSVCIntrin::_InterlockedCompareExchange128_nf;
118 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
119 return MSVCIntrin::_InterlockedCompareExchange128_rel;
120 case clang::AArch64::BI_InterlockedOr8_acq:
121 case clang::AArch64::BI_InterlockedOr16_acq:
122 case clang::AArch64::BI_InterlockedOr_acq:
123 case clang::AArch64::BI_InterlockedOr64_acq:
124 return MSVCIntrin::_InterlockedOr_acq;
125 case clang::AArch64::BI_InterlockedOr8_rel:
126 case clang::AArch64::BI_InterlockedOr16_rel:
127 case clang::AArch64::BI_InterlockedOr_rel:
128 case clang::AArch64::BI_InterlockedOr64_rel:
129 return MSVCIntrin::_InterlockedOr_rel;
130 case clang::AArch64::BI_InterlockedOr8_nf:
131 case clang::AArch64::BI_InterlockedOr16_nf:
132 case clang::AArch64::BI_InterlockedOr_nf:
133 case clang::AArch64::BI_InterlockedOr64_nf:
134 return MSVCIntrin::_InterlockedOr_nf;
135 case clang::AArch64::BI_InterlockedXor8_acq:
136 case clang::AArch64::BI_InterlockedXor16_acq:
137 case clang::AArch64::BI_InterlockedXor_acq:
138 case clang::AArch64::BI_InterlockedXor64_acq:
139 return MSVCIntrin::_InterlockedXor_acq;
140 case clang::AArch64::BI_InterlockedXor8_rel:
141 case clang::AArch64::BI_InterlockedXor16_rel:
142 case clang::AArch64::BI_InterlockedXor_rel:
143 case clang::AArch64::BI_InterlockedXor64_rel:
144 return MSVCIntrin::_InterlockedXor_rel;
145 case clang::AArch64::BI_InterlockedXor8_nf:
146 case clang::AArch64::BI_InterlockedXor16_nf:
147 case clang::AArch64::BI_InterlockedXor_nf:
148 case clang::AArch64::BI_InterlockedXor64_nf:
149 return MSVCIntrin::_InterlockedXor_nf;
150 case clang::AArch64::BI_InterlockedAnd8_acq:
151 case clang::AArch64::BI_InterlockedAnd16_acq:
152 case clang::AArch64::BI_InterlockedAnd_acq:
153 case clang::AArch64::BI_InterlockedAnd64_acq:
154 return MSVCIntrin::_InterlockedAnd_acq;
155 case clang::AArch64::BI_InterlockedAnd8_rel:
156 case clang::AArch64::BI_InterlockedAnd16_rel:
157 case clang::AArch64::BI_InterlockedAnd_rel:
158 case clang::AArch64::BI_InterlockedAnd64_rel:
159 return MSVCIntrin::_InterlockedAnd_rel;
160 case clang::AArch64::BI_InterlockedAnd8_nf:
161 case clang::AArch64::BI_InterlockedAnd16_nf:
162 case clang::AArch64::BI_InterlockedAnd_nf:
163 case clang::AArch64::BI_InterlockedAnd64_nf:
164 return MSVCIntrin::_InterlockedAnd_nf;
165 case clang::AArch64::BI_InterlockedIncrement16_acq:
166 case clang::AArch64::BI_InterlockedIncrement_acq:
167 case clang::AArch64::BI_InterlockedIncrement64_acq:
168 return MSVCIntrin::_InterlockedIncrement_acq;
169 case clang::AArch64::BI_InterlockedIncrement16_rel:
170 case clang::AArch64::BI_InterlockedIncrement_rel:
171 case clang::AArch64::BI_InterlockedIncrement64_rel:
172 return MSVCIntrin::_InterlockedIncrement_rel;
173 case clang::AArch64::BI_InterlockedIncrement16_nf:
174 case clang::AArch64::BI_InterlockedIncrement_nf:
175 case clang::AArch64::BI_InterlockedIncrement64_nf:
176 return MSVCIntrin::_InterlockedIncrement_nf;
177 case clang::AArch64::BI_InterlockedDecrement16_acq:
178 case clang::AArch64::BI_InterlockedDecrement_acq:
179 case clang::AArch64::BI_InterlockedDecrement64_acq:
180 return MSVCIntrin::_InterlockedDecrement_acq;
181 case clang::AArch64::BI_InterlockedDecrement16_rel:
182 case clang::AArch64::BI_InterlockedDecrement_rel:
183 case clang::AArch64::BI_InterlockedDecrement64_rel:
184 return MSVCIntrin::_InterlockedDecrement_rel;
185 case clang::AArch64::BI_InterlockedDecrement16_nf:
186 case clang::AArch64::BI_InterlockedDecrement_nf:
187 case clang::AArch64::BI_InterlockedDecrement64_nf:
188 return MSVCIntrin::_InterlockedDecrement_nf;
189 }
190 llvm_unreachable("must return from switch");
191}
192
193static std::optional<CodeGenFunction::MSVCIntrin>
194translateArmToMsvcIntrin(unsigned BuiltinID) {
195 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
196 switch (BuiltinID) {
197 default:
198 return std::nullopt;
199 case clang::ARM::BI_BitScanForward:
200 case clang::ARM::BI_BitScanForward64:
201 return MSVCIntrin::_BitScanForward;
202 case clang::ARM::BI_BitScanReverse:
203 case clang::ARM::BI_BitScanReverse64:
204 return MSVCIntrin::_BitScanReverse;
205 case clang::ARM::BI_InterlockedAnd64:
206 return MSVCIntrin::_InterlockedAnd;
207 case clang::ARM::BI_InterlockedExchange64:
208 return MSVCIntrin::_InterlockedExchange;
209 case clang::ARM::BI_InterlockedExchangeAdd64:
210 return MSVCIntrin::_InterlockedExchangeAdd;
211 case clang::ARM::BI_InterlockedExchangeSub64:
212 return MSVCIntrin::_InterlockedExchangeSub;
213 case clang::ARM::BI_InterlockedOr64:
214 return MSVCIntrin::_InterlockedOr;
215 case clang::ARM::BI_InterlockedXor64:
216 return MSVCIntrin::_InterlockedXor;
217 case clang::ARM::BI_InterlockedDecrement64:
218 return MSVCIntrin::_InterlockedDecrement;
219 case clang::ARM::BI_InterlockedIncrement64:
220 return MSVCIntrin::_InterlockedIncrement;
221 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
222 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
223 case clang::ARM::BI_InterlockedExchangeAdd_acq:
224 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
225 return MSVCIntrin::_InterlockedExchangeAdd_acq;
226 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
227 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
228 case clang::ARM::BI_InterlockedExchangeAdd_rel:
229 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
230 return MSVCIntrin::_InterlockedExchangeAdd_rel;
231 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
232 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
233 case clang::ARM::BI_InterlockedExchangeAdd_nf:
234 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
235 return MSVCIntrin::_InterlockedExchangeAdd_nf;
236 case clang::ARM::BI_InterlockedExchange8_acq:
237 case clang::ARM::BI_InterlockedExchange16_acq:
238 case clang::ARM::BI_InterlockedExchange_acq:
239 case clang::ARM::BI_InterlockedExchange64_acq:
240 case clang::ARM::BI_InterlockedExchangePointer_acq:
241 return MSVCIntrin::_InterlockedExchange_acq;
242 case clang::ARM::BI_InterlockedExchange8_rel:
243 case clang::ARM::BI_InterlockedExchange16_rel:
244 case clang::ARM::BI_InterlockedExchange_rel:
245 case clang::ARM::BI_InterlockedExchange64_rel:
246 case clang::ARM::BI_InterlockedExchangePointer_rel:
247 return MSVCIntrin::_InterlockedExchange_rel;
248 case clang::ARM::BI_InterlockedExchange8_nf:
249 case clang::ARM::BI_InterlockedExchange16_nf:
250 case clang::ARM::BI_InterlockedExchange_nf:
251 case clang::ARM::BI_InterlockedExchange64_nf:
252 case clang::ARM::BI_InterlockedExchangePointer_nf:
253 return MSVCIntrin::_InterlockedExchange_nf;
254 case clang::ARM::BI_InterlockedCompareExchange8_acq:
255 case clang::ARM::BI_InterlockedCompareExchange16_acq:
256 case clang::ARM::BI_InterlockedCompareExchange_acq:
257 case clang::ARM::BI_InterlockedCompareExchange64_acq:
258 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
259 return MSVCIntrin::_InterlockedCompareExchange_acq;
260 case clang::ARM::BI_InterlockedCompareExchange8_rel:
261 case clang::ARM::BI_InterlockedCompareExchange16_rel:
262 case clang::ARM::BI_InterlockedCompareExchange_rel:
263 case clang::ARM::BI_InterlockedCompareExchange64_rel:
264 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
265 return MSVCIntrin::_InterlockedCompareExchange_rel;
266 case clang::ARM::BI_InterlockedCompareExchange8_nf:
267 case clang::ARM::BI_InterlockedCompareExchange16_nf:
268 case clang::ARM::BI_InterlockedCompareExchange_nf:
269 case clang::ARM::BI_InterlockedCompareExchange64_nf:
270 return MSVCIntrin::_InterlockedCompareExchange_nf;
271 case clang::ARM::BI_InterlockedOr8_acq:
272 case clang::ARM::BI_InterlockedOr16_acq:
273 case clang::ARM::BI_InterlockedOr_acq:
274 case clang::ARM::BI_InterlockedOr64_acq:
275 return MSVCIntrin::_InterlockedOr_acq;
276 case clang::ARM::BI_InterlockedOr8_rel:
277 case clang::ARM::BI_InterlockedOr16_rel:
278 case clang::ARM::BI_InterlockedOr_rel:
279 case clang::ARM::BI_InterlockedOr64_rel:
280 return MSVCIntrin::_InterlockedOr_rel;
281 case clang::ARM::BI_InterlockedOr8_nf:
282 case clang::ARM::BI_InterlockedOr16_nf:
283 case clang::ARM::BI_InterlockedOr_nf:
284 case clang::ARM::BI_InterlockedOr64_nf:
285 return MSVCIntrin::_InterlockedOr_nf;
286 case clang::ARM::BI_InterlockedXor8_acq:
287 case clang::ARM::BI_InterlockedXor16_acq:
288 case clang::ARM::BI_InterlockedXor_acq:
289 case clang::ARM::BI_InterlockedXor64_acq:
290 return MSVCIntrin::_InterlockedXor_acq;
291 case clang::ARM::BI_InterlockedXor8_rel:
292 case clang::ARM::BI_InterlockedXor16_rel:
293 case clang::ARM::BI_InterlockedXor_rel:
294 case clang::ARM::BI_InterlockedXor64_rel:
295 return MSVCIntrin::_InterlockedXor_rel;
296 case clang::ARM::BI_InterlockedXor8_nf:
297 case clang::ARM::BI_InterlockedXor16_nf:
298 case clang::ARM::BI_InterlockedXor_nf:
299 case clang::ARM::BI_InterlockedXor64_nf:
300 return MSVCIntrin::_InterlockedXor_nf;
301 case clang::ARM::BI_InterlockedAnd8_acq:
302 case clang::ARM::BI_InterlockedAnd16_acq:
303 case clang::ARM::BI_InterlockedAnd_acq:
304 case clang::ARM::BI_InterlockedAnd64_acq:
305 return MSVCIntrin::_InterlockedAnd_acq;
306 case clang::ARM::BI_InterlockedAnd8_rel:
307 case clang::ARM::BI_InterlockedAnd16_rel:
308 case clang::ARM::BI_InterlockedAnd_rel:
309 case clang::ARM::BI_InterlockedAnd64_rel:
310 return MSVCIntrin::_InterlockedAnd_rel;
311 case clang::ARM::BI_InterlockedAnd8_nf:
312 case clang::ARM::BI_InterlockedAnd16_nf:
313 case clang::ARM::BI_InterlockedAnd_nf:
314 case clang::ARM::BI_InterlockedAnd64_nf:
315 return MSVCIntrin::_InterlockedAnd_nf;
316 case clang::ARM::BI_InterlockedIncrement16_acq:
317 case clang::ARM::BI_InterlockedIncrement_acq:
318 case clang::ARM::BI_InterlockedIncrement64_acq:
319 return MSVCIntrin::_InterlockedIncrement_acq;
320 case clang::ARM::BI_InterlockedIncrement16_rel:
321 case clang::ARM::BI_InterlockedIncrement_rel:
322 case clang::ARM::BI_InterlockedIncrement64_rel:
323 return MSVCIntrin::_InterlockedIncrement_rel;
324 case clang::ARM::BI_InterlockedIncrement16_nf:
325 case clang::ARM::BI_InterlockedIncrement_nf:
326 case clang::ARM::BI_InterlockedIncrement64_nf:
327 return MSVCIntrin::_InterlockedIncrement_nf;
328 case clang::ARM::BI_InterlockedDecrement16_acq:
329 case clang::ARM::BI_InterlockedDecrement_acq:
330 case clang::ARM::BI_InterlockedDecrement64_acq:
331 return MSVCIntrin::_InterlockedDecrement_acq;
332 case clang::ARM::BI_InterlockedDecrement16_rel:
333 case clang::ARM::BI_InterlockedDecrement_rel:
334 case clang::ARM::BI_InterlockedDecrement64_rel:
335 return MSVCIntrin::_InterlockedDecrement_rel;
336 case clang::ARM::BI_InterlockedDecrement16_nf:
337 case clang::ARM::BI_InterlockedDecrement_nf:
338 case clang::ARM::BI_InterlockedDecrement64_nf:
339 return MSVCIntrin::_InterlockedDecrement_nf;
340 }
341 llvm_unreachable("must return from switch");
342}
343
344// Emit an intrinsic where all operands are of the same type as the result.
345// Depending on mode, this may be a constrained floating-point intrinsic.
346static Value *emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
347 unsigned IntrinsicID,
348 unsigned ConstrainedIntrinsicID,
349 llvm::Type *Ty,
350 ArrayRef<Value *> Args) {
351 Function *F;
352 if (CGF.Builder.getIsFPConstrained())
353 F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Ty);
354 else
355 F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Ty);
356
357 if (CGF.Builder.getIsFPConstrained())
358 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args);
359
360 return CGF.Builder.CreateCall(Callee: F, Args);
361}
362
363static llvm::FixedVectorType *GetNeonType(CodeGenFunction *CGF,
364 NeonTypeFlags TypeFlags,
365 bool HasFastHalfType = true,
366 bool V1Ty = false,
367 bool AllowBFloatArgsAndRet = true) {
368 int IsQuad = TypeFlags.isQuad();
369 switch (TypeFlags.getEltType()) {
370 case NeonTypeFlags::Int8:
371 case NeonTypeFlags::Poly8:
372 case NeonTypeFlags::MFloat8:
373 return llvm::FixedVectorType::get(ElementType: CGF->Int8Ty, NumElts: V1Ty ? 1 : (8 << IsQuad));
374 case NeonTypeFlags::Int16:
375 case NeonTypeFlags::Poly16:
376 return llvm::FixedVectorType::get(ElementType: CGF->Int16Ty, NumElts: V1Ty ? 1 : (4 << IsQuad));
377 case NeonTypeFlags::BFloat16:
378 if (AllowBFloatArgsAndRet)
379 return llvm::FixedVectorType::get(ElementType: CGF->BFloatTy, NumElts: V1Ty ? 1 : (4 << IsQuad));
380 return llvm::FixedVectorType::get(ElementType: CGF->Int16Ty, NumElts: V1Ty ? 1 : (4 << IsQuad));
381 case NeonTypeFlags::Float16:
382 if (HasFastHalfType)
383 return llvm::FixedVectorType::get(ElementType: CGF->HalfTy, NumElts: V1Ty ? 1 : (4 << IsQuad));
384 return llvm::FixedVectorType::get(ElementType: CGF->Int16Ty, NumElts: V1Ty ? 1 : (4 << IsQuad));
385 case NeonTypeFlags::Int32:
386 return llvm::FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: V1Ty ? 1 : (2 << IsQuad));
387 case NeonTypeFlags::Int64:
388 case NeonTypeFlags::Poly64:
389 return llvm::FixedVectorType::get(ElementType: CGF->Int64Ty, NumElts: V1Ty ? 1 : (1 << IsQuad));
390 case NeonTypeFlags::Poly128:
391 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
392 // There is a lot of i128 and f128 API missing.
393 // so we use v16i8 to represent poly128 and get pattern matched.
394 return llvm::FixedVectorType::get(ElementType: CGF->Int8Ty, NumElts: 16);
395 case NeonTypeFlags::Float32:
396 return llvm::FixedVectorType::get(ElementType: CGF->FloatTy, NumElts: V1Ty ? 1 : (2 << IsQuad));
397 case NeonTypeFlags::Float64:
398 return llvm::FixedVectorType::get(ElementType: CGF->DoubleTy, NumElts: V1Ty ? 1 : (1 << IsQuad));
399 }
400 llvm_unreachable("Unknown vector element type!");
401}
402
403static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
404 NeonTypeFlags IntTypeFlags) {
405 int IsQuad = IntTypeFlags.isQuad();
406 switch (IntTypeFlags.getEltType()) {
407 case NeonTypeFlags::Int16:
408 return llvm::FixedVectorType::get(ElementType: CGF->HalfTy, NumElts: (4 << IsQuad));
409 case NeonTypeFlags::Int32:
410 return llvm::FixedVectorType::get(ElementType: CGF->FloatTy, NumElts: (2 << IsQuad));
411 case NeonTypeFlags::Int64:
412 return llvm::FixedVectorType::get(ElementType: CGF->DoubleTy, NumElts: (1 << IsQuad));
413 default:
414 llvm_unreachable("Type can't be converted to floating-point!");
415 }
416}
417
418Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C,
419 const ElementCount &Count) {
420 Value *SV = llvm::ConstantVector::getSplat(EC: Count, Elt: C);
421 return Builder.CreateShuffleVector(V1: V, V2: V, Mask: SV, Name: "lane");
422}
423
424Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
425 ElementCount EC = cast<llvm::VectorType>(Val: V->getType())->getElementCount();
426 return EmitNeonSplat(V, C, Count: EC);
427}
428
429Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
430 const char *name,
431 unsigned shift, bool rightshift) {
432 unsigned j = 0;
433 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
434 ai != ae; ++ai, ++j) {
435 if (F->isConstrainedFPIntrinsic())
436 if (ai->getType()->isMetadataTy())
437 continue;
438 if (shift > 0 && shift == j)
439 Ops[j] = EmitNeonShiftVector(V: Ops[j], Ty: ai->getType(), negateForRightShift: rightshift);
440 else
441 Ops[j] = Builder.CreateBitCast(V: Ops[j], DestTy: ai->getType(), Name: name);
442 }
443
444 if (F->isConstrainedFPIntrinsic())
445 return Builder.CreateConstrainedFPCall(Callee: F, Args: Ops, Name: name);
446 return Builder.CreateCall(Callee: F, Args: Ops, Name: name);
447}
448
449Value *CodeGenFunction::EmitFP8NeonCall(unsigned IID,
450 ArrayRef<llvm::Type *> Tys,
451 SmallVectorImpl<Value *> &Ops,
452 const CallExpr *E, const char *name) {
453 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_set_fpmr),
454 Args: Ops.pop_back_val());
455 return EmitNeonCall(F: CGM.getIntrinsic(IID, Tys), Ops, name);
456}
457
458llvm::Value *CodeGenFunction::EmitFP8NeonFDOTCall(
459 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
460 SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
461
462 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
463 RetTy->getPrimitiveSizeInBits();
464 llvm::Type *Tys[] = {llvm::FixedVectorType::get(ElementType: RetTy, NumElts: ElemCount),
465 Ops[1]->getType()};
466 if (ExtendLaneArg) {
467 auto *VT = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16);
468 Ops[2] = Builder.CreateInsertVector(DstType: VT, SrcVec: PoisonValue::get(T: VT), SubVec: Ops[2],
469 Idx: uint64_t(0));
470 }
471 return EmitFP8NeonCall(IID, Tys, Ops, E, name);
472}
473
474llvm::Value *CodeGenFunction::EmitFP8NeonFMLACall(
475 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
476 SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
477
478 if (ExtendLaneArg) {
479 auto *VT = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16);
480 Ops[2] = Builder.CreateInsertVector(DstType: VT, SrcVec: PoisonValue::get(T: VT), SubVec: Ops[2],
481 Idx: uint64_t(0));
482 }
483 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
484 RetTy->getPrimitiveSizeInBits();
485 return EmitFP8NeonCall(IID, Tys: {llvm::FixedVectorType::get(ElementType: RetTy, NumElts: ElemCount)},
486 Ops, E, name);
487}
488
489Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
490 bool neg) {
491 int SV = cast<ConstantInt>(Val: V)->getSExtValue();
492 return ConstantInt::getSigned(Ty, V: neg ? -SV : SV);
493}
494
495Value *CodeGenFunction::EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
496 llvm::Type *Ty1, bool Extract,
497 SmallVectorImpl<llvm::Value *> &Ops,
498 const CallExpr *E,
499 const char *name) {
500 llvm::Type *Tys[] = {Ty0, Ty1};
501 if (Extract) {
502 // Op[0] is mfloat8x16_t, but the intrinsic converts only the lower part of
503 // the vector.
504 Tys[1] = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 8);
505 Ops[0] = Builder.CreateExtractVector(DstType: Tys[1], SrcVec: Ops[0], Idx: uint64_t(0));
506 }
507 return EmitFP8NeonCall(IID, Tys, Ops, E, name);
508}
509
510// Right-shift a vector by a constant.
511Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
512 llvm::Type *Ty, bool usgn,
513 const char *name) {
514 llvm::VectorType *VTy = cast<llvm::VectorType>(Val: Ty);
515
516 int ShiftAmt = cast<ConstantInt>(Val: Shift)->getSExtValue();
517 int EltSize = VTy->getScalarSizeInBits();
518
519 Vec = Builder.CreateBitCast(V: Vec, DestTy: Ty);
520
521 // lshr/ashr are undefined when the shift amount is equal to the vector
522 // element size.
523 if (ShiftAmt == EltSize) {
524 if (usgn) {
525 // Right-shifting an unsigned value by its size yields 0.
526 return llvm::ConstantAggregateZero::get(Ty: VTy);
527 } else {
528 // Right-shifting a signed value by its size is equivalent
529 // to a shift of size-1.
530 --ShiftAmt;
531 Shift = ConstantInt::get(Ty: VTy->getElementType(), V: ShiftAmt);
532 }
533 }
534
535 Shift = EmitNeonShiftVector(V: Shift, Ty, neg: false);
536 if (usgn)
537 return Builder.CreateLShr(LHS: Vec, RHS: Shift, Name: name);
538 return Builder.CreateAShr(LHS: Vec, RHS: Shift, Name: name);
539}
540
541// clang-format off
542static const ARMNeonVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
543 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
544 NEONMAP0(splat_lane_v),
545 NEONMAP0(splat_laneq_v),
546 NEONMAP0(splatq_lane_v),
547 NEONMAP0(splatq_laneq_v),
548 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
549 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
550 NEONMAP1(vabs_v, arm_neon_vabs, 0),
551 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
552 NEONMAP0(vadd_v),
553 NEONMAP0(vaddhn_v),
554 NEONMAP0(vaddq_v),
555 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
556 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
557 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
558 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
559 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
560 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
561 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
562 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
563 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
564 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
565 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
566 NEONMAP1(vcadd_rot270_f16, arm_neon_vcadd_rot270, Add1ArgType),
567 NEONMAP1(vcadd_rot270_f32, arm_neon_vcadd_rot270, Add1ArgType),
568 NEONMAP1(vcadd_rot90_f16, arm_neon_vcadd_rot90, Add1ArgType),
569 NEONMAP1(vcadd_rot90_f32, arm_neon_vcadd_rot90, Add1ArgType),
570 NEONMAP1(vcaddq_rot270_f16, arm_neon_vcadd_rot270, Add1ArgType),
571 NEONMAP1(vcaddq_rot270_f32, arm_neon_vcadd_rot270, Add1ArgType),
572 NEONMAP1(vcaddq_rot270_f64, arm_neon_vcadd_rot270, Add1ArgType),
573 NEONMAP1(vcaddq_rot90_f16, arm_neon_vcadd_rot90, Add1ArgType),
574 NEONMAP1(vcaddq_rot90_f32, arm_neon_vcadd_rot90, Add1ArgType),
575 NEONMAP1(vcaddq_rot90_f64, arm_neon_vcadd_rot90, Add1ArgType),
576 NEONMAP1(vcage_v, arm_neon_vacge, 0),
577 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
578 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
579 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
580 NEONMAP1(vcale_v, arm_neon_vacge, 0),
581 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
582 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
583 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
584 NEONMAP0(vceqz_v),
585 NEONMAP0(vceqzq_v),
586 NEONMAP0(vcgez_v),
587 NEONMAP0(vcgezq_v),
588 NEONMAP0(vcgtz_v),
589 NEONMAP0(vcgtzq_v),
590 NEONMAP0(vclez_v),
591 NEONMAP0(vclezq_v),
592 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
593 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
594 NEONMAP0(vcltz_v),
595 NEONMAP0(vcltzq_v),
596 NEONMAP1(vclz_v, ctlz, Add1ArgType),
597 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
598 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
599 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
600 NEONMAP0(vcvt_f16_s16),
601 NEONMAP0(vcvt_f16_u16),
602 NEONMAP0(vcvt_f32_v),
603 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
604 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
605 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
606 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
607 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
608 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
609 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
610 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
611 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
612 NEONMAP0(vcvt_s16_f16),
613 NEONMAP0(vcvt_s32_v),
614 NEONMAP0(vcvt_s64_v),
615 NEONMAP0(vcvt_u16_f16),
616 NEONMAP0(vcvt_u32_v),
617 NEONMAP0(vcvt_u64_v),
618 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
619 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
620 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
621 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
622 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
623 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
624 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
625 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
626 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
627 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
628 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
629 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
630 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
631 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
632 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
633 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
634 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
635 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
636 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
637 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
638 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
639 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
640 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
641 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
642 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
643 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
644 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
645 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
646 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
647 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
648 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
649 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
650 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
651 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
652 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
653 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
654 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
655 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
656 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
657 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
658 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
659 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
660 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
661 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
662 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
663 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
664 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
665 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
666 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
667 NEONMAP0(vcvtq_f16_s16),
668 NEONMAP0(vcvtq_f16_u16),
669 NEONMAP0(vcvtq_f32_v),
670 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
671 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
672 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
673 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
674 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
675 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
676 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
677 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
678 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
679 NEONMAP0(vcvtq_s16_f16),
680 NEONMAP0(vcvtq_s32_v),
681 NEONMAP0(vcvtq_s64_v),
682 NEONMAP0(vcvtq_u16_f16),
683 NEONMAP0(vcvtq_u32_v),
684 NEONMAP0(vcvtq_u64_v),
685 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
686 NEONMAP1(vdot_u32, arm_neon_udot, 0),
687 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
688 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
689 NEONMAP0(vext_v),
690 NEONMAP0(vextq_v),
691 NEONMAP0(vfma_v),
692 NEONMAP0(vfmaq_v),
693 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
694 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
695 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
696 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
697 NEONMAP0(vld1_dup_v),
698 NEONMAP1(vld1_v, arm_neon_vld1, 0),
699 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
700 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
701 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
702 NEONMAP0(vld1q_dup_v),
703 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
704 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
705 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
706 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
707 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
708 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
709 NEONMAP1(vld2_v, arm_neon_vld2, 0),
710 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
711 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
712 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
713 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
714 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
715 NEONMAP1(vld3_v, arm_neon_vld3, 0),
716 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
717 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
718 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
719 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
720 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
721 NEONMAP1(vld4_v, arm_neon_vld4, 0),
722 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
723 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
724 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
725 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
726 NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
727 NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
728 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
729 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
730 NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
731 NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
732 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
733 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
734 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
735 NEONMAP0(vmovl_v),
736 NEONMAP0(vmovn_v),
737 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
738 NEONMAP0(vmull_v),
739 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
740 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
741 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
742 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
743 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
744 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
745 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
746 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
747 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
748 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
749 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
750 NEONMAP2(vqadd_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
751 NEONMAP2(vqaddq_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
752 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
753 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
754 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
755 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
756 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
757 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
758 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
759 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
760 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
761 NEONMAP1(vqrdmlah_s16, arm_neon_vqrdmlah, Add1ArgType),
762 NEONMAP1(vqrdmlah_s32, arm_neon_vqrdmlah, Add1ArgType),
763 NEONMAP1(vqrdmlahq_s16, arm_neon_vqrdmlah, Add1ArgType),
764 NEONMAP1(vqrdmlahq_s32, arm_neon_vqrdmlah, Add1ArgType),
765 NEONMAP1(vqrdmlsh_s16, arm_neon_vqrdmlsh, Add1ArgType),
766 NEONMAP1(vqrdmlsh_s32, arm_neon_vqrdmlsh, Add1ArgType),
767 NEONMAP1(vqrdmlshq_s16, arm_neon_vqrdmlsh, Add1ArgType),
768 NEONMAP1(vqrdmlshq_s32, arm_neon_vqrdmlsh, Add1ArgType),
769 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
770 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
771 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
772 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
773 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
774 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
775 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
776 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
777 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
778 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
779 NEONMAP2(vqsub_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
780 NEONMAP2(vqsubq_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
781 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
782 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
783 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
784 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
785 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
786 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
787 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
788 NEONMAP1(vrnd_v, trunc, Add1ArgType),
789 NEONMAP1(vrnda_v, round, Add1ArgType),
790 NEONMAP1(vrndaq_v, round, Add1ArgType),
791 NEONMAP0(vrndi_v),
792 NEONMAP0(vrndiq_v),
793 NEONMAP1(vrndm_v, floor, Add1ArgType),
794 NEONMAP1(vrndmq_v, floor, Add1ArgType),
795 NEONMAP1(vrndn_v, roundeven, Add1ArgType),
796 NEONMAP1(vrndnq_v, roundeven, Add1ArgType),
797 NEONMAP1(vrndp_v, ceil, Add1ArgType),
798 NEONMAP1(vrndpq_v, ceil, Add1ArgType),
799 NEONMAP1(vrndq_v, trunc, Add1ArgType),
800 NEONMAP1(vrndx_v, rint, Add1ArgType),
801 NEONMAP1(vrndxq_v, rint, Add1ArgType),
802 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
803 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
804 NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
805 NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
806 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
807 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
808 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
809 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
810 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
811 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
812 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
813 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
814 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
815 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
816 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
817 NEONMAP0(vshl_n_v),
818 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
819 NEONMAP0(vshll_n_v),
820 NEONMAP0(vshlq_n_v),
821 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
822 NEONMAP0(vshr_n_v),
823 NEONMAP0(vshrn_n_v),
824 NEONMAP0(vshrq_n_v),
825 NEONMAP1(vst1_v, arm_neon_vst1, 0),
826 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
827 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
828 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
829 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
830 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
831 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
832 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
833 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
834 NEONMAP1(vst2_v, arm_neon_vst2, 0),
835 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
836 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
837 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
838 NEONMAP1(vst3_v, arm_neon_vst3, 0),
839 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
840 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
841 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
842 NEONMAP1(vst4_v, arm_neon_vst4, 0),
843 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
844 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
845 NEONMAP0(vsubhn_v),
846 NEONMAP0(vtrn_v),
847 NEONMAP0(vtrnq_v),
848 NEONMAP0(vtst_v),
849 NEONMAP0(vtstq_v),
850 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
851 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
852 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
853 NEONMAP0(vuzp_v),
854 NEONMAP0(vuzpq_v),
855 NEONMAP0(vzip_v),
856 NEONMAP0(vzipq_v)
857};
858
859// clang-format on
860
861// Some intrinsics are equivalent for codegen.
862static const std::pair<unsigned, unsigned> NEONEquivalentIntrinsicMap[] = {
863 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
864 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
865 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
866 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
867 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
868 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
869 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
870 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
871 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
872 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
873 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
874 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
875 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
876 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
877 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
878 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
879 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
880 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
881 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
882 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
883 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
884 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
885 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
886 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
887 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
888 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
889 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
890 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
891 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
892 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
893 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
894 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
895 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
896 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
897 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
898 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
899 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
900 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
901 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
902 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
903 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
904 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
905 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
906 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
907 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
908 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
909 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
910 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
911 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
912 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
913 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
914 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
915 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
916 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
917 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
918 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
919 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
920 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
921 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
922 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
923 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
924 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
925 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
926 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
927 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
928 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
929 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
930 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
931 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
932 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
933 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
934 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
935 // The mangling rules cause us to have one ID for each type for vldap1(q)_lane
936 // and vstl1(q)_lane, but codegen is equivalent for all of them. Choose an
937 // arbitrary one to be handled as tha canonical variation.
938 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
939 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
940 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
941 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
942 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
943 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
944 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
945 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
946 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
947 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
948 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
949 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
950};
951
952#undef NEONMAP0
953#undef NEONMAP1
954#undef NEONMAP2
955
956#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
957 {SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
958
959#define SVEMAP2(NameBase, TypeModifier) \
960 {SVE::BI__builtin_sve_##NameBase, 0, TypeModifier}
961static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SVEIntrinsicMap[] = {
962#define GET_SVE_LLVM_INTRINSIC_MAP
963#include "clang/Basic/arm_sve_builtin_cg.inc"
964#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
965#undef GET_SVE_LLVM_INTRINSIC_MAP
966};
967
968#undef SVEMAP1
969#undef SVEMAP2
970
971#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
972 {SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
973
974#define SMEMAP2(NameBase, TypeModifier) \
975 {SME::BI__builtin_sme_##NameBase, 0, TypeModifier}
976static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SMEIntrinsicMap[] = {
977#define GET_SME_LLVM_INTRINSIC_MAP
978#include "clang/Basic/arm_sme_builtin_cg.inc"
979#undef GET_SME_LLVM_INTRINSIC_MAP
980};
981
982#undef SMEMAP1
983#undef SMEMAP2
984
985static bool NEONSIMDIntrinsicsProvenSorted = false;
986
987static bool AArch64SIMDIntrinsicsProvenSorted = false;
988static bool AArch64SISDIntrinsicsProvenSorted = false;
989static bool AArch64SVEIntrinsicsProvenSorted = false;
990static bool AArch64SMEIntrinsicsProvenSorted = false;
991
992// Check if Builtin `BuiltinId` is present in `IntrinsicMap`. If yes, returns
993// the corresponding info struct.
994template <typename IntrinsicInfo>
995static const IntrinsicInfo *
996findARMVectorIntrinsicInMap(ArrayRef<IntrinsicInfo> IntrinsicMap,
997 unsigned BuiltinID, bool &MapProvenSorted) {
998
999#ifndef NDEBUG
1000 if (!MapProvenSorted) {
1001 assert(llvm::is_sorted(IntrinsicMap));
1002 MapProvenSorted = true;
1003 }
1004#endif
1005
1006 const IntrinsicInfo *Builtin = llvm::lower_bound(IntrinsicMap, BuiltinID);
1007
1008 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
1009 return Builtin;
1010
1011 return nullptr;
1012}
1013
1014Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
1015 unsigned Modifier,
1016 llvm::Type *ArgType,
1017 const CallExpr *E) {
1018 int VectorSize = 0;
1019 if (Modifier & Use64BitVectors)
1020 VectorSize = 64;
1021 else if (Modifier & Use128BitVectors)
1022 VectorSize = 128;
1023
1024 // Return type.
1025 SmallVector<llvm::Type *, 3> Tys;
1026 if (Modifier & AddRetType) {
1027 llvm::Type *Ty = ConvertType(T: E->getCallReturnType(Ctx: getContext()));
1028 if (Modifier & VectorizeRetType)
1029 Ty = llvm::FixedVectorType::get(
1030 ElementType: Ty, NumElts: VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
1031
1032 Tys.push_back(Elt: Ty);
1033 }
1034
1035 // Arguments.
1036 if (Modifier & VectorizeArgTypes) {
1037 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
1038 ArgType = llvm::FixedVectorType::get(ElementType: ArgType, NumElts: Elts);
1039 }
1040
1041 if (Modifier & (Add1ArgType | Add2ArgTypes))
1042 Tys.push_back(Elt: ArgType);
1043
1044 if (Modifier & Add2ArgTypes)
1045 Tys.push_back(Elt: ArgType);
1046
1047 if (Modifier & InventFloatType)
1048 Tys.push_back(Elt: FloatTy);
1049
1050 return CGM.getIntrinsic(IID: IntrinsicID, Tys);
1051}
1052
1053//===----------------------------------------------------------------------===//
1054// Emit-helpers
1055//===----------------------------------------------------------------------===//
1056static Value *EmitCommonNeonSISDBuiltinExpr(
1057 CodeGenFunction &CGF, const ARMNeonVectorIntrinsicInfo &SISDInfo,
1058 SmallVectorImpl<Value *> &Ops, const CallExpr *E) {
1059 assert(SISDInfo.LLVMIntrinsic && "Generic code assumes a valid intrinsic");
1060
1061 switch (SISDInfo.BuiltinID) {
1062 case NEON::BI__builtin_neon_vcled_s64:
1063 case NEON::BI__builtin_neon_vcled_u64:
1064 case NEON::BI__builtin_neon_vcles_f32:
1065 case NEON::BI__builtin_neon_vcled_f64:
1066 case NEON::BI__builtin_neon_vcltd_s64:
1067 case NEON::BI__builtin_neon_vcltd_u64:
1068 case NEON::BI__builtin_neon_vclts_f32:
1069 case NEON::BI__builtin_neon_vcltd_f64:
1070 case NEON::BI__builtin_neon_vcales_f32:
1071 case NEON::BI__builtin_neon_vcaled_f64:
1072 case NEON::BI__builtin_neon_vcalts_f32:
1073 case NEON::BI__builtin_neon_vcaltd_f64:
1074 // Only one direction of comparisons actually exist, cmle is actually a cmge
1075 // with swapped operands. The table gives us the right intrinsic but we
1076 // still need to do the swap.
1077 std::swap(a&: Ops[0], b&: Ops[1]);
1078 break;
1079 }
1080
1081 // Use fptosi.sat/fptoui.sat unless under strict FP.
1082 unsigned LLVMIntrinsic = SISDInfo.LLVMIntrinsic;
1083 if (!CGF.Builder.getIsFPConstrained()) {
1084 if (LLVMIntrinsic == Intrinsic::aarch64_neon_fcvtzs)
1085 LLVMIntrinsic = Intrinsic::fptosi_sat;
1086 else if (LLVMIntrinsic == Intrinsic::aarch64_neon_fcvtzu)
1087 LLVMIntrinsic = Intrinsic::fptoui_sat;
1088 }
1089 llvm::Type *ArgTy = CGF.ConvertType(T: E->getArg(Arg: 0)->getType());
1090 Function *F = CGF.LookupNeonLLVMIntrinsic(IntrinsicID: LLVMIntrinsic,
1091 Modifier: SISDInfo.TypeModifier, ArgType: ArgTy, E);
1092
1093 int j = 0;
1094 ConstantInt *C0 = ConstantInt::get(Ty: CGF.SizeTy, V: 0);
1095 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
1096 ai != ae; ++ai, ++j) {
1097 llvm::Type *ArgTy = ai->getType();
1098 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
1099 ArgTy->getPrimitiveSizeInBits())
1100 continue;
1101 assert(
1102 ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy() &&
1103 "Expecting vector LLVM intrinsic type and scalar Clang builtin type!");
1104
1105 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
1106 // it before inserting.
1107 Ops[j] = CGF.Builder.CreateTruncOrBitCast(
1108 V: Ops[j], DestTy: cast<llvm::VectorType>(Val: ArgTy)->getElementType());
1109 Ops[j] =
1110 CGF.Builder.CreateInsertElement(Vec: PoisonValue::get(T: ArgTy), NewElt: Ops[j], Idx: C0);
1111 }
1112
1113 Value *Result = CGF.EmitNeonCall(F, Ops, name: SISDInfo.NameHint);
1114 llvm::Type *ResultType = CGF.ConvertType(T: E->getType());
1115 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
1116 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
1117 return CGF.Builder.CreateExtractElement(Vec: Result, Idx: C0);
1118
1119 return CGF.Builder.CreateBitCast(V: Result, DestTy: ResultType, Name: SISDInfo.NameHint);
1120}
1121
1122Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
1123 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
1124 const char *NameHint, unsigned Modifier, const CallExpr *E,
1125 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1,
1126 llvm::Triple::ArchType Arch) {
1127
1128 // Extract the trailing immediate argument that encodes the type discriminator
1129 // for this overloaded intrinsic.
1130 // TODO: Move to the parent code that takes care of argument processing.
1131 const Expr *Arg = E->getArg(Arg: E->getNumArgs() - 1);
1132 std::optional<llvm::APSInt> NeonTypeConst =
1133 Arg->getIntegerConstantExpr(Ctx: getContext());
1134 if (!NeonTypeConst)
1135 return nullptr;
1136
1137 // Determine the type of this overloaded NEON intrinsic.
1138 NeonTypeFlags Type(NeonTypeConst->getZExtValue());
1139 const bool Usgn = Type.isUnsigned();
1140 const bool Quad = Type.isQuad();
1141 const bool Floating = Type.isFloatingPoint();
1142 const bool HasFastHalfType = getTarget().hasFastHalfType();
1143 const bool AllowBFloatArgsAndRet =
1144 getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
1145
1146 llvm::FixedVectorType *VTy =
1147 GetNeonType(CGF: this, TypeFlags: Type, HasFastHalfType, V1Ty: false, AllowBFloatArgsAndRet);
1148 llvm::Type *Ty = VTy;
1149 if (!Ty)
1150 return nullptr;
1151
1152 auto getAlignmentValue32 = [&](Address addr) -> Value* {
1153 return Builder.getInt32(C: addr.getAlignment().getQuantity());
1154 };
1155
1156 unsigned Int = LLVMIntrinsic;
1157 if ((Modifier & UnsignedAlts) && !Usgn)
1158 Int = AltLLVMIntrinsic;
1159
1160 switch (BuiltinID) {
1161 default: break;
1162 case NEON::BI__builtin_neon_splat_lane_v:
1163 case NEON::BI__builtin_neon_splat_laneq_v:
1164 case NEON::BI__builtin_neon_splatq_lane_v:
1165 case NEON::BI__builtin_neon_splatq_laneq_v: {
1166 auto NumElements = VTy->getElementCount();
1167 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
1168 NumElements = NumElements * 2;
1169 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
1170 NumElements = NumElements.divideCoefficientBy(RHS: 2);
1171
1172 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: VTy);
1173 return EmitNeonSplat(V: Ops[0], C: cast<ConstantInt>(Val: Ops[1]), Count: NumElements);
1174 }
1175 case NEON::BI__builtin_neon_vpadd_v:
1176 case NEON::BI__builtin_neon_vpaddq_v:
1177 // We don't allow fp/int overloading of intrinsics.
1178 if (VTy->getElementType()->isFloatingPointTy() &&
1179 Int == Intrinsic::aarch64_neon_addp)
1180 Int = Intrinsic::aarch64_neon_faddp;
1181 break;
1182 case NEON::BI__builtin_neon_vabs_v:
1183 case NEON::BI__builtin_neon_vabsq_v:
1184 if (VTy->getElementType()->isFloatingPointTy())
1185 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::fabs, Tys: Ty), Ops, name: "vabs");
1186 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys: Ty), Ops, name: "vabs");
1187 case NEON::BI__builtin_neon_vadd_v:
1188 case NEON::BI__builtin_neon_vaddq_v: {
1189 llvm::Type *VTy = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Quad ? 16 : 8);
1190 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: VTy);
1191 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: VTy);
1192 Ops[0] = Builder.CreateXor(LHS: Ops[0], RHS: Ops[1]);
1193 return Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1194 }
1195 case NEON::BI__builtin_neon_vaddhn_v: {
1196 llvm::FixedVectorType *SrcTy =
1197 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1198
1199 // %sum = add <4 x i32> %lhs, %rhs
1200 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1201 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: SrcTy);
1202 Ops[0] = Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1], Name: "vaddhn");
1203
1204 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
1205 Constant *ShiftAmt =
1206 ConstantInt::get(Ty: SrcTy, V: SrcTy->getScalarSizeInBits() / 2);
1207 Ops[0] = Builder.CreateLShr(LHS: Ops[0], RHS: ShiftAmt, Name: "vaddhn");
1208
1209 // %res = trunc <4 x i32> %high to <4 x i16>
1210 return Builder.CreateTrunc(V: Ops[0], DestTy: VTy, Name: "vaddhn");
1211 }
1212 case NEON::BI__builtin_neon_vcale_v:
1213 case NEON::BI__builtin_neon_vcaleq_v:
1214 case NEON::BI__builtin_neon_vcalt_v:
1215 case NEON::BI__builtin_neon_vcaltq_v:
1216 std::swap(a&: Ops[0], b&: Ops[1]);
1217 [[fallthrough]];
1218 case NEON::BI__builtin_neon_vcage_v:
1219 case NEON::BI__builtin_neon_vcageq_v:
1220 case NEON::BI__builtin_neon_vcagt_v:
1221 case NEON::BI__builtin_neon_vcagtq_v: {
1222 llvm::Type *Ty;
1223 switch (VTy->getScalarSizeInBits()) {
1224 default: llvm_unreachable("unexpected type");
1225 case 32:
1226 Ty = FloatTy;
1227 break;
1228 case 64:
1229 Ty = DoubleTy;
1230 break;
1231 case 16:
1232 Ty = HalfTy;
1233 break;
1234 }
1235 auto *VecFlt = llvm::FixedVectorType::get(ElementType: Ty, NumElts: VTy->getNumElements());
1236 llvm::Type *Tys[] = { VTy, VecFlt };
1237 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1238 return EmitNeonCall(F, Ops, name: NameHint);
1239 }
1240 case NEON::BI__builtin_neon_vceqz_v:
1241 case NEON::BI__builtin_neon_vceqzq_v:
1242 return EmitAArch64CompareBuiltinExpr(
1243 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OEQ : ICmpInst::ICMP_EQ, Name: "vceqz");
1244 case NEON::BI__builtin_neon_vcgez_v:
1245 case NEON::BI__builtin_neon_vcgezq_v:
1246 return EmitAArch64CompareBuiltinExpr(
1247 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OGE : ICmpInst::ICMP_SGE,
1248 Name: "vcgez");
1249 case NEON::BI__builtin_neon_vclez_v:
1250 case NEON::BI__builtin_neon_vclezq_v:
1251 return EmitAArch64CompareBuiltinExpr(
1252 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OLE : ICmpInst::ICMP_SLE,
1253 Name: "vclez");
1254 case NEON::BI__builtin_neon_vcgtz_v:
1255 case NEON::BI__builtin_neon_vcgtzq_v:
1256 return EmitAArch64CompareBuiltinExpr(
1257 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OGT : ICmpInst::ICMP_SGT,
1258 Name: "vcgtz");
1259 case NEON::BI__builtin_neon_vcltz_v:
1260 case NEON::BI__builtin_neon_vcltzq_v:
1261 return EmitAArch64CompareBuiltinExpr(
1262 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OLT : ICmpInst::ICMP_SLT,
1263 Name: "vcltz");
1264 case NEON::BI__builtin_neon_vclz_v:
1265 case NEON::BI__builtin_neon_vclzq_v:
1266 // We generate target-independent intrinsic, which needs a second argument
1267 // for whether or not clz of zero is undefined; on ARM it isn't.
1268 Ops.push_back(Elt: Builder.getInt1(V: getTarget().isCLZForZeroUndef()));
1269 break;
1270 case NEON::BI__builtin_neon_vcvt_f32_v:
1271 case NEON::BI__builtin_neon_vcvtq_f32_v:
1272 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1273 Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float32, false, Quad),
1274 HasFastHalfType);
1275 return Usgn ? Builder.CreateUIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt")
1276 : Builder.CreateSIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt");
1277 case NEON::BI__builtin_neon_vcvt_f16_s16:
1278 case NEON::BI__builtin_neon_vcvt_f16_u16:
1279 case NEON::BI__builtin_neon_vcvtq_f16_s16:
1280 case NEON::BI__builtin_neon_vcvtq_f16_u16:
1281 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1282 Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float16, false, Quad),
1283 HasFastHalfType);
1284 return Usgn ? Builder.CreateUIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt")
1285 : Builder.CreateSIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt");
1286 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
1287 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
1288 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
1289 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
1290 llvm::Type *Tys[2] = { GetFloatNeonType(CGF: this, IntTypeFlags: Type), Ty };
1291 Function *F = CGM.getIntrinsic(IID: Int, Tys);
1292 return EmitNeonCall(F, Ops, name: "vcvt_n");
1293 }
1294 case NEON::BI__builtin_neon_vcvt_n_f32_v:
1295 case NEON::BI__builtin_neon_vcvt_n_f64_v:
1296 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
1297 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
1298 llvm::Type *Tys[2] = { GetFloatNeonType(CGF: this, IntTypeFlags: Type), Ty };
1299 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
1300 Function *F = CGM.getIntrinsic(IID: Int, Tys);
1301 return EmitNeonCall(F, Ops, name: "vcvt_n");
1302 }
1303 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
1304 case NEON::BI__builtin_neon_vcvt_n_s32_v:
1305 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
1306 case NEON::BI__builtin_neon_vcvt_n_u32_v:
1307 case NEON::BI__builtin_neon_vcvt_n_s64_v:
1308 case NEON::BI__builtin_neon_vcvt_n_u64_v:
1309 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
1310 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
1311 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
1312 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
1313 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
1314 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
1315 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
1316 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1317 return EmitNeonCall(F, Ops, name: "vcvt_n");
1318 }
1319 case NEON::BI__builtin_neon_vcvt_s32_v:
1320 case NEON::BI__builtin_neon_vcvt_u32_v:
1321 case NEON::BI__builtin_neon_vcvt_s64_v:
1322 case NEON::BI__builtin_neon_vcvt_u64_v:
1323 case NEON::BI__builtin_neon_vcvt_s16_f16:
1324 case NEON::BI__builtin_neon_vcvt_u16_f16:
1325 case NEON::BI__builtin_neon_vcvtq_s32_v:
1326 case NEON::BI__builtin_neon_vcvtq_u32_v:
1327 case NEON::BI__builtin_neon_vcvtq_s64_v:
1328 case NEON::BI__builtin_neon_vcvtq_u64_v:
1329 case NEON::BI__builtin_neon_vcvtq_s16_f16:
1330 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
1331 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: GetFloatNeonType(CGF: this, IntTypeFlags: Type));
1332 if (Int) {
1333 // AArch64: use fptosi.sat/fptoui.sat unless under strict FP.
1334 if (!Builder.getIsFPConstrained())
1335 Int = Usgn ? Intrinsic::fptoui_sat : Intrinsic::fptosi_sat;
1336 llvm::Type *Tys[2] = {Ty, Ops[0]->getType()};
1337 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtz");
1338 }
1339 // FIXME: ARM uses plain fptoui/fptosi which have UB on out-of-range
1340 // values. These should also use saturating intrinsics.
1341 return Usgn ? Builder.CreateFPToUI(V: Ops[0], DestTy: Ty, Name: "vcvt")
1342 : Builder.CreateFPToSI(V: Ops[0], DestTy: Ty, Name: "vcvt");
1343 }
1344 case NEON::BI__builtin_neon_vcvta_s16_f16:
1345 case NEON::BI__builtin_neon_vcvta_s32_v:
1346 case NEON::BI__builtin_neon_vcvta_s64_v:
1347 case NEON::BI__builtin_neon_vcvta_u16_f16:
1348 case NEON::BI__builtin_neon_vcvta_u32_v:
1349 case NEON::BI__builtin_neon_vcvta_u64_v:
1350 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
1351 case NEON::BI__builtin_neon_vcvtaq_s32_v:
1352 case NEON::BI__builtin_neon_vcvtaq_s64_v:
1353 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
1354 case NEON::BI__builtin_neon_vcvtaq_u32_v:
1355 case NEON::BI__builtin_neon_vcvtaq_u64_v:
1356 case NEON::BI__builtin_neon_vcvtn_s16_f16:
1357 case NEON::BI__builtin_neon_vcvtn_s32_v:
1358 case NEON::BI__builtin_neon_vcvtn_s64_v:
1359 case NEON::BI__builtin_neon_vcvtn_u16_f16:
1360 case NEON::BI__builtin_neon_vcvtn_u32_v:
1361 case NEON::BI__builtin_neon_vcvtn_u64_v:
1362 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
1363 case NEON::BI__builtin_neon_vcvtnq_s32_v:
1364 case NEON::BI__builtin_neon_vcvtnq_s64_v:
1365 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
1366 case NEON::BI__builtin_neon_vcvtnq_u32_v:
1367 case NEON::BI__builtin_neon_vcvtnq_u64_v:
1368 case NEON::BI__builtin_neon_vcvtp_s16_f16:
1369 case NEON::BI__builtin_neon_vcvtp_s32_v:
1370 case NEON::BI__builtin_neon_vcvtp_s64_v:
1371 case NEON::BI__builtin_neon_vcvtp_u16_f16:
1372 case NEON::BI__builtin_neon_vcvtp_u32_v:
1373 case NEON::BI__builtin_neon_vcvtp_u64_v:
1374 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
1375 case NEON::BI__builtin_neon_vcvtpq_s32_v:
1376 case NEON::BI__builtin_neon_vcvtpq_s64_v:
1377 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
1378 case NEON::BI__builtin_neon_vcvtpq_u32_v:
1379 case NEON::BI__builtin_neon_vcvtpq_u64_v:
1380 case NEON::BI__builtin_neon_vcvtm_s16_f16:
1381 case NEON::BI__builtin_neon_vcvtm_s32_v:
1382 case NEON::BI__builtin_neon_vcvtm_s64_v:
1383 case NEON::BI__builtin_neon_vcvtm_u16_f16:
1384 case NEON::BI__builtin_neon_vcvtm_u32_v:
1385 case NEON::BI__builtin_neon_vcvtm_u64_v:
1386 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
1387 case NEON::BI__builtin_neon_vcvtmq_s32_v:
1388 case NEON::BI__builtin_neon_vcvtmq_s64_v:
1389 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
1390 case NEON::BI__builtin_neon_vcvtmq_u32_v:
1391 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
1392 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
1393 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: NameHint);
1394 }
1395 case NEON::BI__builtin_neon_vcvtx_f32_v: {
1396 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
1397 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: NameHint);
1398
1399 }
1400 case NEON::BI__builtin_neon_vext_v:
1401 case NEON::BI__builtin_neon_vextq_v: {
1402 int CV = cast<ConstantInt>(Val: Ops[2])->getSExtValue();
1403 SmallVector<int, 16> Indices;
1404 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1405 Indices.push_back(Elt: i+CV);
1406
1407 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1408 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1409 return Builder.CreateShuffleVector(V1: Ops[0], V2: Ops[1], Mask: Indices, Name: "vext");
1410 }
1411 case NEON::BI__builtin_neon_vfma_v:
1412 case NEON::BI__builtin_neon_vfmaq_v: {
1413 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1414 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1415 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1416
1417 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
1418 return emitCallMaybeConstrainedFPBuiltin(
1419 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
1420 Args: {Ops[1], Ops[2], Ops[0]});
1421 }
1422 case NEON::BI__builtin_neon_vld1_x2_v:
1423 case NEON::BI__builtin_neon_vld1q_x2_v:
1424 case NEON::BI__builtin_neon_vld1_x3_v:
1425 case NEON::BI__builtin_neon_vld1q_x3_v:
1426 case NEON::BI__builtin_neon_vld1_x4_v:
1427 case NEON::BI__builtin_neon_vld1q_x4_v: {
1428 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
1429 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1430 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld1xN");
1431 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
1432 }
1433 case NEON::BI__builtin_neon_vld1_v:
1434 case NEON::BI__builtin_neon_vld1q_v: {
1435 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1436 Ops.push_back(Elt: getAlignmentValue32(PtrOp0));
1437 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "vld1");
1438 }
1439 case NEON::BI__builtin_neon_vld2_v:
1440 case NEON::BI__builtin_neon_vld2q_v:
1441 case NEON::BI__builtin_neon_vld3_v:
1442 case NEON::BI__builtin_neon_vld3q_v:
1443 case NEON::BI__builtin_neon_vld4_v:
1444 case NEON::BI__builtin_neon_vld4q_v:
1445 case NEON::BI__builtin_neon_vld2_dup_v:
1446 case NEON::BI__builtin_neon_vld2q_dup_v:
1447 case NEON::BI__builtin_neon_vld3_dup_v:
1448 case NEON::BI__builtin_neon_vld3q_dup_v:
1449 case NEON::BI__builtin_neon_vld4_dup_v:
1450 case NEON::BI__builtin_neon_vld4q_dup_v: {
1451 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1452 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1453 Value *Align = getAlignmentValue32(PtrOp1);
1454 Ops[1] = Builder.CreateCall(Callee: F, Args: {Ops[1], Align}, Name: NameHint);
1455 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
1456 }
1457 case NEON::BI__builtin_neon_vld1_dup_v:
1458 case NEON::BI__builtin_neon_vld1q_dup_v: {
1459 Value *V = PoisonValue::get(T: Ty);
1460 PtrOp0 = PtrOp0.withElementType(ElemTy: VTy->getElementType());
1461 LoadInst *Ld = Builder.CreateLoad(Addr: PtrOp0);
1462 llvm::Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
1463 Ops[0] = Builder.CreateInsertElement(Vec: V, NewElt: Ld, Idx: CI);
1464 return EmitNeonSplat(V: Ops[0], C: CI);
1465 }
1466 case NEON::BI__builtin_neon_vld2_lane_v:
1467 case NEON::BI__builtin_neon_vld2q_lane_v:
1468 case NEON::BI__builtin_neon_vld3_lane_v:
1469 case NEON::BI__builtin_neon_vld3q_lane_v:
1470 case NEON::BI__builtin_neon_vld4_lane_v:
1471 case NEON::BI__builtin_neon_vld4q_lane_v: {
1472 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1473 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1474 for (unsigned I = 2; I < Ops.size() - 1; ++I)
1475 Ops[I] = Builder.CreateBitCast(V: Ops[I], DestTy: Ty);
1476 Ops.push_back(Elt: getAlignmentValue32(PtrOp1));
1477 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: NameHint);
1478 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
1479 }
1480 case NEON::BI__builtin_neon_vmovl_v: {
1481 llvm::FixedVectorType *DTy =
1482 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1483 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DTy);
1484 if (Usgn)
1485 return Builder.CreateZExt(V: Ops[0], DestTy: Ty, Name: "vmovl");
1486 return Builder.CreateSExt(V: Ops[0], DestTy: Ty, Name: "vmovl");
1487 }
1488 case NEON::BI__builtin_neon_vmovn_v: {
1489 llvm::FixedVectorType *QTy =
1490 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1491 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: QTy);
1492 return Builder.CreateTrunc(V: Ops[0], DestTy: Ty, Name: "vmovn");
1493 }
1494 case NEON::BI__builtin_neon_vmull_v:
1495 // FIXME: the integer vmull operations could be emitted in terms of pure
1496 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
1497 // hoisting the exts outside loops. Until global ISel comes along that can
1498 // see through such movement this leads to bad CodeGen. So we need an
1499 // intrinsic for now.
1500 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
1501 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
1502 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmull");
1503 case NEON::BI__builtin_neon_vpadal_v:
1504 case NEON::BI__builtin_neon_vpadalq_v: {
1505 // The source operand type has twice as many elements of half the size.
1506 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1507 llvm::Type *EltTy =
1508 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: EltBits / 2);
1509 auto *NarrowTy =
1510 llvm::FixedVectorType::get(ElementType: EltTy, NumElts: VTy->getNumElements() * 2);
1511 llvm::Type *Tys[2] = { Ty, NarrowTy };
1512 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: NameHint);
1513 }
1514 case NEON::BI__builtin_neon_vpaddl_v:
1515 case NEON::BI__builtin_neon_vpaddlq_v: {
1516 // The source operand type has twice as many elements of half the size.
1517 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1518 llvm::Type *EltTy = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: EltBits / 2);
1519 auto *NarrowTy =
1520 llvm::FixedVectorType::get(ElementType: EltTy, NumElts: VTy->getNumElements() * 2);
1521 llvm::Type *Tys[2] = { Ty, NarrowTy };
1522 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vpaddl");
1523 }
1524 case NEON::BI__builtin_neon_vqdmlal_v:
1525 case NEON::BI__builtin_neon_vqdmlsl_v: {
1526 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
1527 Ops[1] =
1528 EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys: Ty), Ops&: MulOps, name: "vqdmlal");
1529 Ops.resize(N: 2);
1530 return EmitNeonCall(F: CGM.getIntrinsic(IID: AltLLVMIntrinsic, Tys: Ty), Ops, name: NameHint);
1531 }
1532 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
1533 case NEON::BI__builtin_neon_vqdmulh_lane_v:
1534 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
1535 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
1536 auto *RTy = cast<llvm::FixedVectorType>(Val: Ty);
1537 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
1538 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
1539 RTy = llvm::FixedVectorType::get(ElementType: RTy->getElementType(),
1540 NumElts: RTy->getNumElements() * 2);
1541 llvm::Type *Tys[2] = {
1542 RTy, GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
1543 /*isQuad*/ false))};
1544 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: NameHint);
1545 }
1546 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
1547 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
1548 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
1549 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
1550 llvm::Type *Tys[2] = {
1551 Ty, GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
1552 /*isQuad*/ true))};
1553 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: NameHint);
1554 }
1555 case NEON::BI__builtin_neon_vqshl_n_v:
1556 case NEON::BI__builtin_neon_vqshlq_n_v:
1557 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshl_n",
1558 shift: 1, rightshift: false);
1559 case NEON::BI__builtin_neon_vqshlu_n_v:
1560 case NEON::BI__builtin_neon_vqshluq_n_v:
1561 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshlu_n",
1562 shift: 1, rightshift: false);
1563 case NEON::BI__builtin_neon_vrecpe_v:
1564 case NEON::BI__builtin_neon_vrecpeq_v:
1565 case NEON::BI__builtin_neon_vrsqrte_v:
1566 case NEON::BI__builtin_neon_vrsqrteq_v:
1567 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
1568 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: NameHint);
1569 case NEON::BI__builtin_neon_vrndi_v:
1570 case NEON::BI__builtin_neon_vrndiq_v:
1571 Int = Builder.getIsFPConstrained()
1572 ? Intrinsic::experimental_constrained_nearbyint
1573 : Intrinsic::nearbyint;
1574 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: NameHint);
1575 case NEON::BI__builtin_neon_vrshr_n_v:
1576 case NEON::BI__builtin_neon_vrshrq_n_v:
1577 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrshr_n",
1578 shift: 1, rightshift: true);
1579 case NEON::BI__builtin_neon_vsha512hq_u64:
1580 case NEON::BI__builtin_neon_vsha512h2q_u64:
1581 case NEON::BI__builtin_neon_vsha512su0q_u64:
1582 case NEON::BI__builtin_neon_vsha512su1q_u64: {
1583 Function *F = CGM.getIntrinsic(IID: Int);
1584 return EmitNeonCall(F, Ops, name: "");
1585 }
1586 case NEON::BI__builtin_neon_vshl_n_v:
1587 case NEON::BI__builtin_neon_vshlq_n_v:
1588 Ops[1] = EmitNeonShiftVector(V: Ops[1], Ty, neg: false);
1589 return Builder.CreateShl(LHS: Builder.CreateBitCast(V: Ops[0],DestTy: Ty), RHS: Ops[1],
1590 Name: "vshl_n");
1591 case NEON::BI__builtin_neon_vshll_n_v: {
1592 llvm::FixedVectorType *SrcTy =
1593 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1594 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1595 if (Usgn)
1596 Ops[0] = Builder.CreateZExt(V: Ops[0], DestTy: VTy);
1597 else
1598 Ops[0] = Builder.CreateSExt(V: Ops[0], DestTy: VTy);
1599 Ops[1] = EmitNeonShiftVector(V: Ops[1], Ty: VTy, neg: false);
1600 return Builder.CreateShl(LHS: Ops[0], RHS: Ops[1], Name: "vshll_n");
1601 }
1602 case NEON::BI__builtin_neon_vshrn_n_v: {
1603 llvm::FixedVectorType *SrcTy =
1604 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1605 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1606 Ops[1] = EmitNeonShiftVector(V: Ops[1], Ty: SrcTy, neg: false);
1607 if (Usgn)
1608 Ops[0] = Builder.CreateLShr(LHS: Ops[0], RHS: Ops[1]);
1609 else
1610 Ops[0] = Builder.CreateAShr(LHS: Ops[0], RHS: Ops[1]);
1611 return Builder.CreateTrunc(V: Ops[0], DestTy: Ty, Name: "vshrn_n");
1612 }
1613 case NEON::BI__builtin_neon_vshr_n_v:
1614 case NEON::BI__builtin_neon_vshrq_n_v:
1615 return EmitNeonRShiftImm(Vec: Ops[0], Shift: Ops[1], Ty, usgn: Usgn, name: "vshr_n");
1616 case NEON::BI__builtin_neon_vst1_v:
1617 case NEON::BI__builtin_neon_vst1q_v:
1618 case NEON::BI__builtin_neon_vst2_v:
1619 case NEON::BI__builtin_neon_vst2q_v:
1620 case NEON::BI__builtin_neon_vst3_v:
1621 case NEON::BI__builtin_neon_vst3q_v:
1622 case NEON::BI__builtin_neon_vst4_v:
1623 case NEON::BI__builtin_neon_vst4q_v:
1624 case NEON::BI__builtin_neon_vst2_lane_v:
1625 case NEON::BI__builtin_neon_vst2q_lane_v:
1626 case NEON::BI__builtin_neon_vst3_lane_v:
1627 case NEON::BI__builtin_neon_vst3q_lane_v:
1628 case NEON::BI__builtin_neon_vst4_lane_v:
1629 case NEON::BI__builtin_neon_vst4q_lane_v: {
1630 llvm::Type *Tys[] = {Int8PtrTy, Ty};
1631 Ops.push_back(Elt: getAlignmentValue32(PtrOp0));
1632 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "");
1633 }
1634 case NEON::BI__builtin_neon_vsm3partw1q_u32:
1635 case NEON::BI__builtin_neon_vsm3partw2q_u32:
1636 case NEON::BI__builtin_neon_vsm3ss1q_u32:
1637 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
1638 case NEON::BI__builtin_neon_vsm4eq_u32: {
1639 Function *F = CGM.getIntrinsic(IID: Int);
1640 return EmitNeonCall(F, Ops, name: "");
1641 }
1642 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
1643 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
1644 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
1645 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
1646 Function *F = CGM.getIntrinsic(IID: Int);
1647 Ops[3] = Builder.CreateZExt(V: Ops[3], DestTy: Int64Ty);
1648 return EmitNeonCall(F, Ops, name: "");
1649 }
1650 case NEON::BI__builtin_neon_vst1_x2_v:
1651 case NEON::BI__builtin_neon_vst1q_x2_v:
1652 case NEON::BI__builtin_neon_vst1_x3_v:
1653 case NEON::BI__builtin_neon_vst1q_x3_v:
1654 case NEON::BI__builtin_neon_vst1_x4_v:
1655 case NEON::BI__builtin_neon_vst1q_x4_v: {
1656 // TODO: Currently in AArch32 mode the pointer operand comes first, whereas
1657 // in AArch64 it comes last. We may want to stick to one or another.
1658 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
1659 Arch == llvm::Triple::aarch64_32) {
1660 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
1661 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
1662 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "");
1663 }
1664 llvm::Type *Tys[2] = {DefaultPtrTy, VTy};
1665 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "");
1666 }
1667 case NEON::BI__builtin_neon_vsubhn_v: {
1668 llvm::FixedVectorType *SrcTy =
1669 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1670
1671 // %sum = add <4 x i32> %lhs, %rhs
1672 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1673 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: SrcTy);
1674 Ops[0] = Builder.CreateSub(LHS: Ops[0], RHS: Ops[1], Name: "vsubhn");
1675
1676 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
1677 Constant *ShiftAmt =
1678 ConstantInt::get(Ty: SrcTy, V: SrcTy->getScalarSizeInBits() / 2);
1679 Ops[0] = Builder.CreateLShr(LHS: Ops[0], RHS: ShiftAmt, Name: "vsubhn");
1680
1681 // %res = trunc <4 x i32> %high to <4 x i16>
1682 return Builder.CreateTrunc(V: Ops[0], DestTy: VTy, Name: "vsubhn");
1683 }
1684 case NEON::BI__builtin_neon_vtrn_v:
1685 case NEON::BI__builtin_neon_vtrnq_v: {
1686 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1687 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1688 Value *SV = nullptr;
1689
1690 for (unsigned vi = 0; vi != 2; ++vi) {
1691 SmallVector<int, 16> Indices;
1692 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1693 Indices.push_back(Elt: i+vi);
1694 Indices.push_back(Elt: i+e+vi);
1695 }
1696 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
1697 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vtrn");
1698 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
1699 }
1700 return SV;
1701 }
1702 case NEON::BI__builtin_neon_vtst_v:
1703 case NEON::BI__builtin_neon_vtstq_v: {
1704 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1705 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1706 Ops[0] = Builder.CreateAnd(LHS: Ops[0], RHS: Ops[1]);
1707 Ops[0] = Builder.CreateICmp(P: ICmpInst::ICMP_NE, LHS: Ops[0],
1708 RHS: ConstantAggregateZero::get(Ty));
1709 return Builder.CreateSExt(V: Ops[0], DestTy: Ty, Name: "vtst");
1710 }
1711 case NEON::BI__builtin_neon_vuzp_v:
1712 case NEON::BI__builtin_neon_vuzpq_v: {
1713 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1714 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1715 Value *SV = nullptr;
1716
1717 for (unsigned vi = 0; vi != 2; ++vi) {
1718 SmallVector<int, 16> Indices;
1719 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1720 Indices.push_back(Elt: 2*i+vi);
1721
1722 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
1723 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vuzp");
1724 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
1725 }
1726 return SV;
1727 }
1728 case NEON::BI__builtin_neon_vxarq_u64: {
1729 Function *F = CGM.getIntrinsic(IID: Int);
1730 Ops[2] = Builder.CreateZExt(V: Ops[2], DestTy: Int64Ty);
1731 return EmitNeonCall(F, Ops, name: "");
1732 }
1733 case NEON::BI__builtin_neon_vzip_v:
1734 case NEON::BI__builtin_neon_vzipq_v: {
1735 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1736 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1737 Value *SV = nullptr;
1738
1739 for (unsigned vi = 0; vi != 2; ++vi) {
1740 SmallVector<int, 16> Indices;
1741 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1742 Indices.push_back(Elt: (i + vi*e) >> 1);
1743 Indices.push_back(Elt: ((i + vi*e) >> 1)+e);
1744 }
1745 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
1746 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vzip");
1747 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
1748 }
1749 return SV;
1750 }
1751 case NEON::BI__builtin_neon_vdot_s32:
1752 case NEON::BI__builtin_neon_vdot_u32:
1753 case NEON::BI__builtin_neon_vdotq_s32:
1754 case NEON::BI__builtin_neon_vdotq_u32: {
1755 auto *InputTy =
1756 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1757 llvm::Type *Tys[2] = { Ty, InputTy };
1758 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vdot");
1759 }
1760 case NEON::BI__builtin_neon_vfmlal_low_f16:
1761 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
1762 auto *InputTy =
1763 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1764 llvm::Type *Tys[2] = { Ty, InputTy };
1765 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlal_low");
1766 }
1767 case NEON::BI__builtin_neon_vfmlsl_low_f16:
1768 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
1769 auto *InputTy =
1770 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1771 llvm::Type *Tys[2] = { Ty, InputTy };
1772 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlsl_low");
1773 }
1774 case NEON::BI__builtin_neon_vfmlal_high_f16:
1775 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
1776 auto *InputTy =
1777 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1778 llvm::Type *Tys[2] = { Ty, InputTy };
1779 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlal_high");
1780 }
1781 case NEON::BI__builtin_neon_vfmlsl_high_f16:
1782 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
1783 auto *InputTy =
1784 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1785 llvm::Type *Tys[2] = { Ty, InputTy };
1786 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlsl_high");
1787 }
1788 case NEON::BI__builtin_neon_vmmlaq_s32:
1789 case NEON::BI__builtin_neon_vmmlaq_u32: {
1790 auto *InputTy =
1791 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1792 llvm::Type *Tys[2] = { Ty, InputTy };
1793 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "vmmla");
1794 }
1795 case NEON::BI__builtin_neon_vmmlaq_f16:
1796 case NEON::BI__builtin_neon_vmmlaq_f32_f16: {
1797 auto *InputTy =
1798 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1799 llvm::Type *Tys[2] = {Ty, InputTy};
1800 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fmmla");
1801 }
1802 case NEON::BI__builtin_neon_vusmmlaq_s32: {
1803 auto *InputTy =
1804 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1805 llvm::Type *Tys[2] = { Ty, InputTy };
1806 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vusmmla");
1807 }
1808 case NEON::BI__builtin_neon_vusdot_s32:
1809 case NEON::BI__builtin_neon_vusdotq_s32: {
1810 auto *InputTy =
1811 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1812 llvm::Type *Tys[2] = { Ty, InputTy };
1813 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vusdot");
1814 }
1815 case NEON::BI__builtin_neon_vbfdot_f32:
1816 case NEON::BI__builtin_neon_vbfdotq_f32: {
1817 llvm::Type *InputTy =
1818 llvm::FixedVectorType::get(ElementType: BFloatTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1819 llvm::Type *Tys[2] = { Ty, InputTy };
1820 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vbfdot");
1821 }
1822 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
1823 llvm::Type *Tys[1] = { Ty };
1824 Function *F = CGM.getIntrinsic(IID: Int, Tys);
1825 return EmitNeonCall(F, Ops, name: "vcvtfp2bf");
1826 }
1827
1828 }
1829
1830 assert(Int && "Expected valid intrinsic number");
1831
1832 // Determine the type(s) of this overloaded AArch64 intrinsic.
1833 Function *F = LookupNeonLLVMIntrinsic(IntrinsicID: Int, Modifier, ArgType: Ty, E);
1834
1835 Value *Result = EmitNeonCall(F, Ops, name: NameHint);
1836 llvm::Type *ResultType = ConvertType(T: E->getType());
1837 // AArch64 intrinsic one-element vector type cast to
1838 // scalar type expected by the builtin
1839 return Builder.CreateBitCast(V: Result, DestTy: ResultType, Name: NameHint);
1840}
1841
1842Value *
1843CodeGenFunction::EmitAArch64CompareBuiltinExpr(Value *Op, llvm::Type *Ty,
1844 const CmpInst::Predicate Pred,
1845 const Twine &Name) {
1846
1847 if (isa<FixedVectorType>(Val: Ty)) {
1848 // Vector types are cast to i8 vectors. Recover original type.
1849 Op = Builder.CreateBitCast(V: Op, DestTy: Ty);
1850 }
1851
1852 Constant *zero = Constant::getNullValue(Ty: Op->getType());
1853
1854 if (CmpInst::isFPPredicate(P: Pred)) {
1855 if (Pred == CmpInst::FCMP_OEQ)
1856 Op = Builder.CreateFCmp(P: Pred, LHS: Op, RHS: zero);
1857 else
1858 Op = Builder.CreateFCmpS(P: Pred, LHS: Op, RHS: zero);
1859 } else {
1860 Op = Builder.CreateICmp(P: Pred, LHS: Op, RHS: zero);
1861 }
1862
1863 llvm::Type *ResTy = Ty;
1864 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty))
1865 ResTy = FixedVectorType::get(
1866 ElementType: IntegerType::get(C&: getLLVMContext(), NumBits: VTy->getScalarSizeInBits()),
1867 NumElts: VTy->getNumElements());
1868
1869 return Builder.CreateSExt(V: Op, DestTy: ResTy, Name);
1870}
1871
1872static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
1873 Value *ExtOp, Value *IndexOp,
1874 llvm::Type *ResTy, unsigned IntID,
1875 const char *Name) {
1876 SmallVector<Value *, 2> TblOps;
1877 if (ExtOp)
1878 TblOps.push_back(Elt: ExtOp);
1879
1880 // Build a vector containing sequential number like (0, 1, 2, ..., 15)
1881 SmallVector<int, 16> Indices;
1882 auto *TblTy = cast<llvm::FixedVectorType>(Val: Ops[0]->getType());
1883 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
1884 Indices.push_back(Elt: 2*i);
1885 Indices.push_back(Elt: 2*i+1);
1886 }
1887
1888 int PairPos = 0, End = Ops.size() - 1;
1889 while (PairPos < End) {
1890 TblOps.push_back(Elt: CGF.Builder.CreateShuffleVector(V1: Ops[PairPos],
1891 V2: Ops[PairPos+1], Mask: Indices,
1892 Name));
1893 PairPos += 2;
1894 }
1895
1896 // If there's an odd number of 64-bit lookup table, fill the high 64-bit
1897 // of the 128-bit lookup table with zero.
1898 if (PairPos == End) {
1899 Value *ZeroTbl = ConstantAggregateZero::get(Ty: TblTy);
1900 TblOps.push_back(Elt: CGF.Builder.CreateShuffleVector(V1: Ops[PairPos],
1901 V2: ZeroTbl, Mask: Indices, Name));
1902 }
1903
1904 Function *TblF;
1905 TblOps.push_back(Elt: IndexOp);
1906 TblF = CGF.CGM.getIntrinsic(IID: IntID, Tys: ResTy);
1907
1908 return CGF.EmitNeonCall(F: TblF, Ops&: TblOps, name: Name);
1909}
1910
1911Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
1912 unsigned Value;
1913 switch (BuiltinID) {
1914 default:
1915 return nullptr;
1916 case clang::ARM::BI__builtin_arm_nop:
1917 Value = 0;
1918 break;
1919 case clang::ARM::BI__builtin_arm_yield:
1920 case clang::ARM::BI__yield:
1921 Value = 1;
1922 break;
1923 case clang::ARM::BI__builtin_arm_wfe:
1924 case clang::ARM::BI__wfe:
1925 Value = 2;
1926 break;
1927 case clang::ARM::BI__builtin_arm_wfi:
1928 case clang::ARM::BI__wfi:
1929 Value = 3;
1930 break;
1931 case clang::ARM::BI__builtin_arm_sev:
1932 case clang::ARM::BI__sev:
1933 Value = 4;
1934 break;
1935 case clang::ARM::BI__builtin_arm_sevl:
1936 case clang::ARM::BI__sevl:
1937 Value = 5;
1938 break;
1939 }
1940
1941 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_hint),
1942 Args: llvm::ConstantInt::get(Ty: Int32Ty, V: Value));
1943}
1944
1945enum SpecialRegisterAccessKind {
1946 NormalRead,
1947 VolatileRead,
1948 Write,
1949};
1950
1951// Generates the IR for the read/write special register builtin,
1952// ValueType is the type of the value that is to be written or read,
1953// RegisterType is the type of the register being written to or read from.
1954static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF,
1955 const CallExpr *E,
1956 llvm::Type *RegisterType,
1957 llvm::Type *ValueType,
1958 SpecialRegisterAccessKind AccessKind,
1959 StringRef SysReg = "") {
1960 // write and register intrinsics only support 32, 64 and 128 bit operations.
1961 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
1962 RegisterType->isIntegerTy(128)) &&
1963 "Unsupported size for register.");
1964
1965 CodeGen::CGBuilderTy &Builder = CGF.Builder;
1966 CodeGen::CodeGenModule &CGM = CGF.CGM;
1967 LLVMContext &Context = CGM.getLLVMContext();
1968
1969 if (SysReg.empty()) {
1970 const Expr *SysRegStrExpr = E->getArg(Arg: 0)->IgnoreParenCasts();
1971 SysReg = cast<clang::StringLiteral>(Val: SysRegStrExpr)->getString();
1972 }
1973
1974 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, Str: SysReg) };
1975 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
1976 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
1977
1978 llvm::Type *Types[] = { RegisterType };
1979
1980 bool MixedTypes = RegisterType->isIntegerTy(BitWidth: 64) && ValueType->isIntegerTy(BitWidth: 32);
1981 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
1982 && "Can't fit 64-bit value in 32-bit register");
1983
1984 if (AccessKind != Write) {
1985 assert(AccessKind == NormalRead || AccessKind == VolatileRead);
1986 llvm::Function *F = CGM.getIntrinsic(
1987 IID: AccessKind == VolatileRead ? Intrinsic::read_volatile_register
1988 : Intrinsic::read_register,
1989 Tys: Types);
1990 llvm::Value *Call = Builder.CreateCall(Callee: F, Args: Metadata);
1991
1992 if (MixedTypes)
1993 // Read into 64 bit register and then truncate result to 32 bit.
1994 return Builder.CreateTrunc(V: Call, DestTy: ValueType);
1995
1996 if (ValueType->isPointerTy())
1997 // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*).
1998 return Builder.CreateIntToPtr(V: Call, DestTy: ValueType);
1999
2000 return Call;
2001 }
2002
2003 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::write_register, Tys: Types);
2004 llvm::Value *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
2005 if (MixedTypes) {
2006 // Extend 32 bit write value to 64 bit to pass to write.
2007 ArgValue = Builder.CreateZExt(V: ArgValue, DestTy: RegisterType);
2008 return Builder.CreateCall(Callee: F, Args: { Metadata, ArgValue });
2009 }
2010
2011 if (ValueType->isPointerTy()) {
2012 // Have VoidPtrTy ArgValue but want to return an i32/i64.
2013 ArgValue = Builder.CreatePtrToInt(V: ArgValue, DestTy: RegisterType);
2014 return Builder.CreateCall(Callee: F, Args: { Metadata, ArgValue });
2015 }
2016
2017 return Builder.CreateCall(Callee: F, Args: { Metadata, ArgValue });
2018}
2019
2020static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
2021 const CallExpr *E) {
2022 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2023 CodeGen::CodeGenModule &CGM = CGF.CGM;
2024 SmallVector<llvm::Value *, 4> Ops;
2025
2026 auto getIntArg = [&](unsigned ArgNo) {
2027 Expr::EvalResult Result;
2028 if (!E->getArg(Arg: ArgNo)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
2029 llvm_unreachable("Expected constant argument to range prefetch.");
2030 return Result.Val.getInt().getExtValue();
2031 };
2032
2033 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 0))); /*Addr*/
2034 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 1))); /*Access Kind*/
2035 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 2))); /*Policy*/
2036
2037 if (BuiltinID == clang::AArch64::BI__builtin_arm_range_prefetch_x) {
2038 auto Length = getIntArg(3);
2039 auto Count = getIntArg(4) - 1;
2040 auto Stride = getIntArg(5);
2041 auto Distance = getIntArg(6);
2042
2043 // Map ReuseDistance given in bytes to four bits representing decreasing
2044 // powers of two in the range 512MiB (0b0001) to 32KiB (0b1111). Values
2045 // are rounded up to the nearest power of 2, starting at 32KiB. Any value
2046 // over the maximum is represented by 0 (distance not known).
2047 if (Distance > 0) {
2048 Distance = llvm::Log2_32_Ceil(Value: Distance);
2049 if (Distance < 15)
2050 Distance = 15;
2051 else if (Distance > 29)
2052 Distance = 0;
2053 else
2054 Distance = 30 - Distance;
2055 }
2056
2057 uint64_t Mask22 = (1ULL << 22) - 1;
2058 uint64_t Mask16 = (1ULL << 16) - 1;
2059 uint64_t Metadata = (Distance << 60) | ((Stride & Mask22) << 38) |
2060 ((Count & Mask16) << 22) | (Length & Mask22);
2061
2062 Ops.push_back(Elt: llvm::ConstantInt::get(Ty: Builder.getInt64Ty(), V: Metadata));
2063 } else
2064 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 3)));
2065
2066 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_range_prefetch),
2067 Args: Ops);
2068}
2069
2070static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
2071 unsigned BuiltinID,
2072 const CallExpr *E) {
2073 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2074 CodeGen::CodeGenModule &CGM = CGF.CGM;
2075 Expr::EvalResult Result;
2076 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
2077 llvm_unreachable(
2078 "Expected integer policy argument to atomic store with hint.");
2079
2080 const Expr *Ptr = E->getArg(Arg: 0);
2081 Address Addr = CGF.EmitPointerWithAlignment(Addr: Ptr);
2082 Addr = Addr.withElementType(
2083 ElemTy: CGF.ConvertTypeForMem(T: Ptr->getType()->getPointeeType()));
2084
2085 const Expr *Data = E->getArg(Arg: 1);
2086 Value *DataVal = CGF.EmitToMemory(Value: CGF.EmitScalarExpr(E: Data), Ty: Data->getType());
2087
2088 StoreInst *Store = Builder.CreateStore(Val: DataVal, Addr);
2089 Store->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
2090
2091 AtomicOrdering Ordering;
2092 unsigned OrderingArg = Result.Val.getInt().getExtValue();
2093 assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
2094
2095 switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
2096 default:
2097 llvm_unreachable("Unsupported atomic ordering found.");
2098 case AtomicOrderingCABI::relaxed:
2099 Ordering = AtomicOrdering::Monotonic;
2100 break;
2101 case AtomicOrderingCABI::release:
2102 Ordering = AtomicOrdering::Release;
2103 break;
2104 case AtomicOrderingCABI::seq_cst:
2105 Ordering = AtomicOrdering::SequentiallyConsistent;
2106 break;
2107 }
2108 Store->setAtomic(Ordering);
2109
2110 if (!E->getArg(Arg: 3)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
2111 llvm_unreachable(
2112 "Expected integer hint argument to atomic store with hint.");
2113 unsigned HintArg = Result.Val.getInt().getExtValue();
2114
2115 // Attach the hint if valid
2116 if (toAArch64MemoryHint(I: HintArg) != AArch64MemoryHint::NONE) {
2117 LLVMContext &Ctx = CGM.getLLVMContext();
2118 MDNode *MemHint = MDNode::get(
2119 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "aarch64.mem_hint"),
2120 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: HintArg))});
2121 MDNode *HintNode = MDNode::get(
2122 Context&: CGM.getLLVMContext(),
2123 MDs: {llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)), MemHint});
2124
2125 Store->setMetadata(KindID: llvm::LLVMContext::MD_mem_cache_hint, Node: HintNode);
2126 }
2127
2128 return Store;
2129}
2130
2131Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
2132 const CallExpr *E,
2133 ReturnValueSlot ReturnValue,
2134 llvm::Triple::ArchType Arch) {
2135 if (auto Hint = GetValueForARMHint(BuiltinID))
2136 return Hint;
2137
2138 if (BuiltinID == clang::ARM::BI__emit) {
2139 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
2140 llvm::FunctionType *FTy =
2141 llvm::FunctionType::get(Result: VoidTy, /*Variadic=*/isVarArg: false);
2142
2143 Expr::EvalResult Result;
2144 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
2145 llvm_unreachable("Sema will ensure that the parameter is constant");
2146
2147 llvm::APSInt Value = Result.Val.getInt();
2148 uint64_t ZExtValue = Value.zextOrTrunc(width: IsThumb ? 16 : 32).getZExtValue();
2149
2150 llvm::InlineAsm *Emit =
2151 IsThumb ? InlineAsm::get(Ty: FTy, AsmString: ".inst.n 0x" + utohexstr(X: ZExtValue), Constraints: "",
2152 /*hasSideEffects=*/true)
2153 : InlineAsm::get(Ty: FTy, AsmString: ".inst 0x" + utohexstr(X: ZExtValue), Constraints: "",
2154 /*hasSideEffects=*/true);
2155
2156 return Builder.CreateCall(Callee: Emit);
2157 }
2158
2159 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
2160 Value *Option = EmitScalarExpr(E: E->getArg(Arg: 0));
2161 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_dbg), Args: Option);
2162 }
2163
2164 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
2165 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
2166 Value *RW = EmitScalarExpr(E: E->getArg(Arg: 1));
2167 Value *IsData = EmitScalarExpr(E: E->getArg(Arg: 2));
2168
2169 // Locality is not supported on ARM target
2170 Value *Locality = llvm::ConstantInt::get(Ty: Int32Ty, V: 3);
2171
2172 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
2173 return Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, IsData});
2174 }
2175
2176 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
2177 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2178 return Builder.CreateCall(
2179 Callee: CGM.getIntrinsic(IID: Intrinsic::bitreverse, Tys: Arg->getType()), Args: Arg, Name: "rbit");
2180 }
2181
2182 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
2183 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
2184 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2185 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: Arg->getType());
2186 Value *Res = Builder.CreateCall(Callee: F, Args: {Arg, Builder.getInt1(V: false)});
2187 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
2188 Res = Builder.CreateTrunc(V: Res, DestTy: Builder.getInt32Ty());
2189 return Res;
2190 }
2191
2192
2193 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
2194 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2195 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_cls), Args: Arg, Name: "cls");
2196 }
2197 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
2198 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2199 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_cls64), Args: Arg,
2200 Name: "cls");
2201 }
2202
2203 if (BuiltinID == clang::ARM::BI__clear_cache) {
2204 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
2205 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
2206 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache, Tys: {CGM.DefaultPtrTy});
2207 return Builder.CreateCall(Callee: F, Args: {Begin, End});
2208 }
2209
2210 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
2211 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
2212 Function *F;
2213
2214 switch (BuiltinID) {
2215 default: llvm_unreachable("unexpected builtin");
2216 case clang::ARM::BI__builtin_arm_mcrr:
2217 F = CGM.getIntrinsic(IID: Intrinsic::arm_mcrr);
2218 break;
2219 case clang::ARM::BI__builtin_arm_mcrr2:
2220 F = CGM.getIntrinsic(IID: Intrinsic::arm_mcrr2);
2221 break;
2222 }
2223
2224 // MCRR{2} instruction has 5 operands but
2225 // the intrinsic has 4 because Rt and Rt2
2226 // are represented as a single unsigned 64
2227 // bit integer in the intrinsic definition
2228 // but internally it's represented as 2 32
2229 // bit integers.
2230
2231 Value *Coproc = EmitScalarExpr(E: E->getArg(Arg: 0));
2232 Value *Opc1 = EmitScalarExpr(E: E->getArg(Arg: 1));
2233 Value *RtAndRt2 = EmitScalarExpr(E: E->getArg(Arg: 2));
2234 Value *CRm = EmitScalarExpr(E: E->getArg(Arg: 3));
2235
2236 Value *C1 = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2237 Value *Rt = Builder.CreateTruncOrBitCast(V: RtAndRt2, DestTy: Int32Ty);
2238 Value *Rt2 = Builder.CreateLShr(LHS: RtAndRt2, RHS: C1);
2239 Rt2 = Builder.CreateTruncOrBitCast(V: Rt2, DestTy: Int32Ty);
2240
2241 return Builder.CreateCall(Callee: F, Args: {Coproc, Opc1, Rt, Rt2, CRm});
2242 }
2243
2244 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
2245 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
2246 Function *F;
2247
2248 switch (BuiltinID) {
2249 default: llvm_unreachable("unexpected builtin");
2250 case clang::ARM::BI__builtin_arm_mrrc:
2251 F = CGM.getIntrinsic(IID: Intrinsic::arm_mrrc);
2252 break;
2253 case clang::ARM::BI__builtin_arm_mrrc2:
2254 F = CGM.getIntrinsic(IID: Intrinsic::arm_mrrc2);
2255 break;
2256 }
2257
2258 Value *Coproc = EmitScalarExpr(E: E->getArg(Arg: 0));
2259 Value *Opc1 = EmitScalarExpr(E: E->getArg(Arg: 1));
2260 Value *CRm = EmitScalarExpr(E: E->getArg(Arg: 2));
2261 Value *RtAndRt2 = Builder.CreateCall(Callee: F, Args: {Coproc, Opc1, CRm});
2262
2263 // Returns an unsigned 64 bit integer, represented
2264 // as two 32 bit integers.
2265
2266 Value *Rt = Builder.CreateExtractValue(Agg: RtAndRt2, Idxs: 1);
2267 Value *Rt1 = Builder.CreateExtractValue(Agg: RtAndRt2, Idxs: 0);
2268 Rt = Builder.CreateZExt(V: Rt, DestTy: Int64Ty);
2269 Rt1 = Builder.CreateZExt(V: Rt1, DestTy: Int64Ty);
2270
2271 Value *ShiftCast = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2272 RtAndRt2 = Builder.CreateShl(LHS: Rt, RHS: ShiftCast, Name: "shl", HasNUW: true);
2273 RtAndRt2 = Builder.CreateOr(LHS: RtAndRt2, RHS: Rt1);
2274
2275 return Builder.CreateBitCast(V: RtAndRt2, DestTy: ConvertType(T: E->getType()));
2276 }
2277
2278 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
2279 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2280 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
2281 getContext().getTypeSize(T: E->getType()) == 64) ||
2282 BuiltinID == clang::ARM::BI__ldrexd) {
2283 Function *F;
2284
2285 switch (BuiltinID) {
2286 default: llvm_unreachable("unexpected builtin");
2287 case clang::ARM::BI__builtin_arm_ldaex:
2288 F = CGM.getIntrinsic(IID: Intrinsic::arm_ldaexd);
2289 break;
2290 case clang::ARM::BI__builtin_arm_ldrexd:
2291 case clang::ARM::BI__builtin_arm_ldrex:
2292 case clang::ARM::BI__ldrexd:
2293 F = CGM.getIntrinsic(IID: Intrinsic::arm_ldrexd);
2294 break;
2295 }
2296
2297 Value *LdPtr = EmitScalarExpr(E: E->getArg(Arg: 0));
2298 Value *Val = Builder.CreateCall(Callee: F, Args: LdPtr, Name: "ldrexd");
2299
2300 Value *Val0 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
2301 Value *Val1 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
2302 Val0 = Builder.CreateZExt(V: Val0, DestTy: Int64Ty);
2303 Val1 = Builder.CreateZExt(V: Val1, DestTy: Int64Ty);
2304
2305 Value *ShiftCst = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2306 Val = Builder.CreateShl(LHS: Val0, RHS: ShiftCst, Name: "shl", HasNUW: true /* nuw */);
2307 Val = Builder.CreateOr(LHS: Val, RHS: Val1);
2308 return Builder.CreateBitCast(V: Val, DestTy: ConvertType(T: E->getType()));
2309 }
2310
2311 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2312 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
2313 Value *LoadAddr = EmitScalarExpr(E: E->getArg(Arg: 0));
2314
2315 QualType Ty = E->getType();
2316 llvm::Type *RealResTy = ConvertType(T: Ty);
2317 llvm::Type *IntTy =
2318 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
2319
2320 Function *F = CGM.getIntrinsic(
2321 IID: BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
2322 : Intrinsic::arm_ldrex,
2323 Tys: DefaultPtrTy);
2324 CallInst *Val = Builder.CreateCall(Callee: F, Args: LoadAddr, Name: "ldrex");
2325 Val->addParamAttr(
2326 ArgNo: 0, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: IntTy));
2327
2328 if (RealResTy->isPointerTy())
2329 return Builder.CreateIntToPtr(V: Val, DestTy: RealResTy);
2330 else {
2331 llvm::Type *IntResTy = llvm::IntegerType::get(
2332 C&: getLLVMContext(), NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: RealResTy));
2333 return Builder.CreateBitCast(V: Builder.CreateTruncOrBitCast(V: Val, DestTy: IntResTy),
2334 DestTy: RealResTy);
2335 }
2336 }
2337
2338 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
2339 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
2340 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
2341 getContext().getTypeSize(T: E->getArg(Arg: 0)->getType()) == 64)) {
2342 Function *F = CGM.getIntrinsic(
2343 IID: BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
2344 : Intrinsic::arm_strexd);
2345 llvm::Type *STy = llvm::StructType::get(elt1: Int32Ty, elts: Int32Ty);
2346
2347 Address Tmp = CreateMemTempWithoutCast(T: E->getArg(Arg: 0)->getType());
2348 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
2349 Builder.CreateStore(Val, Addr: Tmp);
2350
2351 Address LdPtr = Tmp.withElementType(ElemTy: STy);
2352 Val = Builder.CreateLoad(Addr: LdPtr);
2353
2354 Value *Arg0 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
2355 Value *Arg1 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
2356 Value *StPtr = EmitScalarExpr(E: E->getArg(Arg: 1));
2357 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1, StPtr}, Name: "strexd");
2358 }
2359
2360 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
2361 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
2362 Value *StoreVal = EmitScalarExpr(E: E->getArg(Arg: 0));
2363 Value *StoreAddr = EmitScalarExpr(E: E->getArg(Arg: 1));
2364
2365 QualType Ty = E->getArg(Arg: 0)->getType();
2366 llvm::Type *StoreTy =
2367 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
2368
2369 if (StoreVal->getType()->isPointerTy())
2370 StoreVal = Builder.CreatePtrToInt(V: StoreVal, DestTy: Int32Ty);
2371 else {
2372 llvm::Type *IntTy = llvm::IntegerType::get(
2373 C&: getLLVMContext(),
2374 NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: StoreVal->getType()));
2375 StoreVal = Builder.CreateBitCast(V: StoreVal, DestTy: IntTy);
2376 StoreVal = Builder.CreateZExtOrBitCast(V: StoreVal, DestTy: Int32Ty);
2377 }
2378
2379 Function *F = CGM.getIntrinsic(
2380 IID: BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
2381 : Intrinsic::arm_strex,
2382 Tys: StoreAddr->getType());
2383
2384 CallInst *CI = Builder.CreateCall(Callee: F, Args: {StoreVal, StoreAddr}, Name: "strex");
2385 CI->addParamAttr(
2386 ArgNo: 1, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: StoreTy));
2387 return CI;
2388 }
2389
2390 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
2391 Function *F = CGM.getIntrinsic(IID: Intrinsic::arm_clrex);
2392 return Builder.CreateCall(Callee: F);
2393 }
2394
2395 // CRC32
2396 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
2397 switch (BuiltinID) {
2398 case clang::ARM::BI__builtin_arm_crc32b:
2399 CRCIntrinsicID = Intrinsic::arm_crc32b; break;
2400 case clang::ARM::BI__builtin_arm_crc32cb:
2401 CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
2402 case clang::ARM::BI__builtin_arm_crc32h:
2403 CRCIntrinsicID = Intrinsic::arm_crc32h; break;
2404 case clang::ARM::BI__builtin_arm_crc32ch:
2405 CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
2406 case clang::ARM::BI__builtin_arm_crc32w:
2407 case clang::ARM::BI__builtin_arm_crc32d:
2408 CRCIntrinsicID = Intrinsic::arm_crc32w; break;
2409 case clang::ARM::BI__builtin_arm_crc32cw:
2410 case clang::ARM::BI__builtin_arm_crc32cd:
2411 CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
2412 }
2413
2414 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
2415 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
2416 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
2417
2418 // crc32{c,}d intrinsics are implemented as two calls to crc32{c,}w
2419 // intrinsics, hence we need different codegen for these cases.
2420 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
2421 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
2422 Value *C1 = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2423 Value *Arg1a = Builder.CreateTruncOrBitCast(V: Arg1, DestTy: Int32Ty);
2424 Value *Arg1b = Builder.CreateLShr(LHS: Arg1, RHS: C1);
2425 Arg1b = Builder.CreateTruncOrBitCast(V: Arg1b, DestTy: Int32Ty);
2426
2427 Function *F = CGM.getIntrinsic(IID: CRCIntrinsicID);
2428 Value *Res = Builder.CreateCall(Callee: F, Args: {Arg0, Arg1a});
2429 return Builder.CreateCall(Callee: F, Args: {Res, Arg1b});
2430 } else {
2431 Arg1 = Builder.CreateZExtOrBitCast(V: Arg1, DestTy: Int32Ty);
2432
2433 Function *F = CGM.getIntrinsic(IID: CRCIntrinsicID);
2434 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1});
2435 }
2436 }
2437
2438 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2439 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2440 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2441 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
2442 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
2443 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
2444
2445 SpecialRegisterAccessKind AccessKind = Write;
2446 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2447 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2448 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
2449 AccessKind = VolatileRead;
2450
2451 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2452 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
2453
2454 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2455 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
2456
2457 llvm::Type *ValueType;
2458 llvm::Type *RegisterType;
2459 if (IsPointerBuiltin) {
2460 ValueType = VoidPtrTy;
2461 RegisterType = Int32Ty;
2462 } else if (Is64Bit) {
2463 ValueType = RegisterType = Int64Ty;
2464 } else {
2465 ValueType = RegisterType = Int32Ty;
2466 }
2467
2468 return EmitSpecialRegisterBuiltin(CGF&: *this, E, RegisterType, ValueType,
2469 AccessKind);
2470 }
2471
2472 if (BuiltinID == ARM::BI__builtin_sponentry) {
2473 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: AllocaInt8PtrTy);
2474 return Builder.CreateCall(Callee: F);
2475 }
2476
2477 // Handle MSVC intrinsics before argument evaluation to prevent double
2478 // evaluation.
2479 if (std::optional<MSVCIntrin> MsvcIntId = translateArmToMsvcIntrin(BuiltinID))
2480 return EmitMSVCBuiltinExpr(BuiltinID: *MsvcIntId, E);
2481
2482 // Deal with MVE builtins
2483 if (Value *Result = EmitARMMVEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
2484 return Result;
2485 // Handle CDE builtins
2486 if (Value *Result = EmitARMCDEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
2487 return Result;
2488
2489 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
2490 auto It = llvm::find_if(Range: NEONEquivalentIntrinsicMap, P: [BuiltinID](auto &P) {
2491 return P.first == BuiltinID;
2492 });
2493 if (It != end(arr: NEONEquivalentIntrinsicMap))
2494 BuiltinID = It->second;
2495
2496 // Find out if any arguments are required to be integer constant
2497 // expressions.
2498 unsigned ICEArguments = 0;
2499 ASTContext::GetBuiltinTypeError Error;
2500 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
2501 assert(Error == ASTContext::GE_None && "Should not codegen an error");
2502
2503 auto getAlignmentValue32 = [&](Address addr) -> Value* {
2504 return Builder.getInt32(C: addr.getAlignment().getQuantity());
2505 };
2506
2507 Address PtrOp0 = Address::invalid();
2508 Address PtrOp1 = Address::invalid();
2509 SmallVector<Value*, 4> Ops;
2510 bool HasExtraArg = CodeGenUtils::hasExtraNeonArgument(BuiltinID);
2511 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
2512 for (unsigned i = 0, e = NumArgs; i != e; i++) {
2513 if (i == 0) {
2514 switch (BuiltinID) {
2515 case NEON::BI__builtin_neon_vld1_v:
2516 case NEON::BI__builtin_neon_vld1q_v:
2517 case NEON::BI__builtin_neon_vld1q_lane_v:
2518 case NEON::BI__builtin_neon_vld1_lane_v:
2519 case NEON::BI__builtin_neon_vld1_dup_v:
2520 case NEON::BI__builtin_neon_vld1q_dup_v:
2521 case NEON::BI__builtin_neon_vst1_v:
2522 case NEON::BI__builtin_neon_vst1q_v:
2523 case NEON::BI__builtin_neon_vst1q_lane_v:
2524 case NEON::BI__builtin_neon_vst1_lane_v:
2525 case NEON::BI__builtin_neon_vst2_v:
2526 case NEON::BI__builtin_neon_vst2q_v:
2527 case NEON::BI__builtin_neon_vst2_lane_v:
2528 case NEON::BI__builtin_neon_vst2q_lane_v:
2529 case NEON::BI__builtin_neon_vst3_v:
2530 case NEON::BI__builtin_neon_vst3q_v:
2531 case NEON::BI__builtin_neon_vst3_lane_v:
2532 case NEON::BI__builtin_neon_vst3q_lane_v:
2533 case NEON::BI__builtin_neon_vst4_v:
2534 case NEON::BI__builtin_neon_vst4q_v:
2535 case NEON::BI__builtin_neon_vst4_lane_v:
2536 case NEON::BI__builtin_neon_vst4q_lane_v:
2537 // Get the alignment for the argument in addition to the value;
2538 // we'll use it later.
2539 PtrOp0 = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
2540 Ops.push_back(Elt: PtrOp0.emitRawPointer(CGF&: *this));
2541 continue;
2542 }
2543 }
2544 if (i == 1) {
2545 switch (BuiltinID) {
2546 case NEON::BI__builtin_neon_vld2_v:
2547 case NEON::BI__builtin_neon_vld2q_v:
2548 case NEON::BI__builtin_neon_vld3_v:
2549 case NEON::BI__builtin_neon_vld3q_v:
2550 case NEON::BI__builtin_neon_vld4_v:
2551 case NEON::BI__builtin_neon_vld4q_v:
2552 case NEON::BI__builtin_neon_vld2_lane_v:
2553 case NEON::BI__builtin_neon_vld2q_lane_v:
2554 case NEON::BI__builtin_neon_vld3_lane_v:
2555 case NEON::BI__builtin_neon_vld3q_lane_v:
2556 case NEON::BI__builtin_neon_vld4_lane_v:
2557 case NEON::BI__builtin_neon_vld4q_lane_v:
2558 case NEON::BI__builtin_neon_vld2_dup_v:
2559 case NEON::BI__builtin_neon_vld2q_dup_v:
2560 case NEON::BI__builtin_neon_vld3_dup_v:
2561 case NEON::BI__builtin_neon_vld3q_dup_v:
2562 case NEON::BI__builtin_neon_vld4_dup_v:
2563 case NEON::BI__builtin_neon_vld4q_dup_v:
2564 // Get the alignment for the argument in addition to the value;
2565 // we'll use it later.
2566 PtrOp1 = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
2567 Ops.push_back(Elt: PtrOp1.emitRawPointer(CGF&: *this));
2568 continue;
2569 }
2570 }
2571
2572 Ops.push_back(Elt: EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E));
2573 }
2574
2575 switch (BuiltinID) {
2576 default: break;
2577
2578 case NEON::BI__builtin_neon_vget_lane_i8:
2579 case NEON::BI__builtin_neon_vget_lane_i16:
2580 case NEON::BI__builtin_neon_vget_lane_i32:
2581 case NEON::BI__builtin_neon_vget_lane_i64:
2582 case NEON::BI__builtin_neon_vget_lane_bf16:
2583 case NEON::BI__builtin_neon_vget_lane_f32:
2584 case NEON::BI__builtin_neon_vgetq_lane_i8:
2585 case NEON::BI__builtin_neon_vgetq_lane_i16:
2586 case NEON::BI__builtin_neon_vgetq_lane_i32:
2587 case NEON::BI__builtin_neon_vgetq_lane_i64:
2588 case NEON::BI__builtin_neon_vgetq_lane_bf16:
2589 case NEON::BI__builtin_neon_vgetq_lane_f32:
2590 case NEON::BI__builtin_neon_vduph_lane_bf16:
2591 case NEON::BI__builtin_neon_vduph_laneq_bf16:
2592 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
2593
2594 case NEON::BI__builtin_neon_vrndns_f32: {
2595 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2596 llvm::Type *Tys[] = {Arg->getType()};
2597 Function *F = CGM.getIntrinsic(IID: Intrinsic::roundeven, Tys);
2598 return Builder.CreateCall(Callee: F, Args: {Arg}, Name: "vrndn"); }
2599
2600 case NEON::BI__builtin_neon_vset_lane_i8:
2601 case NEON::BI__builtin_neon_vset_lane_i16:
2602 case NEON::BI__builtin_neon_vset_lane_i32:
2603 case NEON::BI__builtin_neon_vset_lane_i64:
2604 case NEON::BI__builtin_neon_vset_lane_bf16:
2605 case NEON::BI__builtin_neon_vset_lane_f32:
2606 case NEON::BI__builtin_neon_vsetq_lane_i8:
2607 case NEON::BI__builtin_neon_vsetq_lane_i16:
2608 case NEON::BI__builtin_neon_vsetq_lane_i32:
2609 case NEON::BI__builtin_neon_vsetq_lane_i64:
2610 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2611 case NEON::BI__builtin_neon_vsetq_lane_f32:
2612 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
2613
2614 case NEON::BI__builtin_neon_vsha1h_u32:
2615 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1h), Ops,
2616 name: "vsha1h");
2617 case NEON::BI__builtin_neon_vsha1cq_u32:
2618 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1c), Ops,
2619 name: "vsha1h");
2620 case NEON::BI__builtin_neon_vsha1pq_u32:
2621 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1p), Ops,
2622 name: "vsha1h");
2623 case NEON::BI__builtin_neon_vsha1mq_u32:
2624 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1m), Ops,
2625 name: "vsha1h");
2626
2627 case NEON::BI__builtin_neon_vcvth_bf16_f32:
2628 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vcvtbfp2bf), Ops,
2629 name: "vcvtbfp2bf");
2630 case NEON::BI__builtin_neon_vcvt_f16_f32:
2631 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vcvtfp2hf), Ops,
2632 name: "vcvtfp2hf");
2633 case NEON::BI__builtin_neon_vcvt_f32_f16:
2634 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vcvthf2fp), Ops,
2635 name: "vcvthf2fp");
2636
2637 // The ARM _MoveToCoprocessor builtins put the input register value as
2638 // the first argument, but the LLVM intrinsic expects it as the third one.
2639 case clang::ARM::BI_MoveToCoprocessor:
2640 case clang::ARM::BI_MoveToCoprocessor2: {
2641 Function *F = CGM.getIntrinsic(IID: BuiltinID == clang::ARM::BI_MoveToCoprocessor
2642 ? Intrinsic::arm_mcr
2643 : Intrinsic::arm_mcr2);
2644 return Builder.CreateCall(Callee: F, Args: {Ops[1], Ops[2], Ops[0],
2645 Ops[3], Ops[4], Ops[5]});
2646 }
2647 }
2648
2649 // Get the last argument, which specifies the vector type.
2650 assert(HasExtraArg);
2651 const Expr *Arg = E->getArg(Arg: E->getNumArgs()-1);
2652 std::optional<llvm::APSInt> Result =
2653 Arg->getIntegerConstantExpr(Ctx: getContext());
2654 if (!Result)
2655 return nullptr;
2656
2657 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
2658 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
2659 // Determine the overloaded type of this builtin.
2660 llvm::Type *Ty;
2661 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
2662 Ty = FloatTy;
2663 else
2664 Ty = DoubleTy;
2665
2666 // Determine whether this is an unsigned conversion or not.
2667 bool usgn = Result->getZExtValue() == 1;
2668 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
2669
2670 // Call the appropriate intrinsic.
2671 Function *F = CGM.getIntrinsic(IID: Int, Tys: Ty);
2672 return Builder.CreateCall(Callee: F, Args: Ops, Name: "vcvtr");
2673 }
2674
2675 // Determine the type of this overloaded NEON intrinsic.
2676 NeonTypeFlags Type = Result->getZExtValue();
2677 bool usgn = Type.isUnsigned();
2678 bool rightShift = false;
2679
2680 llvm::FixedVectorType *VTy =
2681 GetNeonType(CGF: this, TypeFlags: Type, HasFastHalfType: getTarget().hasFastHalfType(), V1Ty: false,
2682 AllowBFloatArgsAndRet: getTarget().hasBFloat16Type());
2683 llvm::Type *Ty = VTy;
2684 if (!Ty)
2685 return nullptr;
2686
2687 // Many NEON builtins have identical semantics and uses in ARM and
2688 // AArch64. Emit these in a single function.
2689 auto IntrinsicMap = ArrayRef(ARMSIMDIntrinsicMap);
2690 const ARMNeonVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap(
2691 IntrinsicMap, BuiltinID, MapProvenSorted&: NEONSIMDIntrinsicsProvenSorted);
2692 if (Builtin)
2693 return EmitCommonNeonBuiltinExpr(
2694 BuiltinID: Builtin->BuiltinID, LLVMIntrinsic: Builtin->LLVMIntrinsic, AltLLVMIntrinsic: Builtin->AltLLVMIntrinsic,
2695 NameHint: Builtin->NameHint, Modifier: Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
2696
2697 unsigned Int;
2698 switch (BuiltinID) {
2699 default: return nullptr;
2700 case NEON::BI__builtin_neon_vld1q_lane_v:
2701 // Handle 64-bit integer elements as a special case. Use shuffles of
2702 // one-element vectors to avoid poor code for i64 in the backend.
2703 if (VTy->getElementType()->isIntegerTy(BitWidth: 64)) {
2704 // Extract the other lane.
2705 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2706 int Lane = cast<ConstantInt>(Val: Ops[2])->getZExtValue();
2707 Value *SV = llvm::ConstantVector::get(V: ConstantInt::get(Ty: Int32Ty, V: 1-Lane));
2708 Ops[1] = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[1], Mask: SV);
2709 // Load the value as a one-element vector.
2710 Ty = llvm::FixedVectorType::get(ElementType: VTy->getElementType(), NumElts: 1);
2711 llvm::Type *Tys[] = {Ty, Int8PtrTy};
2712 Function *F = CGM.getIntrinsic(IID: Intrinsic::arm_neon_vld1, Tys);
2713 Value *Align = getAlignmentValue32(PtrOp0);
2714 Value *Ld = Builder.CreateCall(Callee: F, Args: {Ops[0], Align});
2715 // Combine them.
2716 int Indices[] = {1 - Lane, Lane};
2717 return Builder.CreateShuffleVector(V1: Ops[1], V2: Ld, Mask: Indices, Name: "vld1q_lane");
2718 }
2719 [[fallthrough]];
2720 case NEON::BI__builtin_neon_vld1_lane_v: {
2721 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2722 PtrOp0 = PtrOp0.withElementType(ElemTy: VTy->getElementType());
2723 Value *Ld = Builder.CreateLoad(Addr: PtrOp0);
2724 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ld, Idx: Ops[2], Name: "vld1_lane");
2725 }
2726 case NEON::BI__builtin_neon_vqrshrn_n_v:
2727 Int =
2728 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
2729 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqrshrn_n",
2730 shift: 1, rightshift: true);
2731 case NEON::BI__builtin_neon_vqrshrun_n_v:
2732 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vqrshiftnsu, Tys: Ty),
2733 Ops, name: "vqrshrun_n", shift: 1, rightshift: true);
2734 case NEON::BI__builtin_neon_vqshrn_n_v:
2735 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
2736 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshrn_n",
2737 shift: 1, rightshift: true);
2738 case NEON::BI__builtin_neon_vqshrun_n_v:
2739 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vqshiftnsu, Tys: Ty),
2740 Ops, name: "vqshrun_n", shift: 1, rightshift: true);
2741 case NEON::BI__builtin_neon_vrecpe_v:
2742 case NEON::BI__builtin_neon_vrecpeq_v:
2743 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vrecpe, Tys: Ty),
2744 Ops, name: "vrecpe");
2745 case NEON::BI__builtin_neon_vrshrn_n_v:
2746 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vrshiftn, Tys: Ty),
2747 Ops, name: "vrshrn_n", shift: 1, rightshift: true);
2748 case NEON::BI__builtin_neon_vrsra_n_v:
2749 case NEON::BI__builtin_neon_vrsraq_n_v:
2750 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
2751 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2752 Ops[2] = EmitNeonShiftVector(V: Ops[2], Ty, neg: true);
2753 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
2754 Ops[1] = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Int, Tys: Ty), Args: {Ops[1], Ops[2]});
2755 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1], Name: "vrsra_n");
2756 case NEON::BI__builtin_neon_vsri_n_v:
2757 case NEON::BI__builtin_neon_vsriq_n_v:
2758 rightShift = true;
2759 [[fallthrough]];
2760 case NEON::BI__builtin_neon_vsli_n_v:
2761 case NEON::BI__builtin_neon_vsliq_n_v:
2762 Ops[2] = EmitNeonShiftVector(V: Ops[2], Ty, neg: rightShift);
2763 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vshiftins, Tys: Ty),
2764 Ops, name: "vsli_n");
2765 case NEON::BI__builtin_neon_vsra_n_v:
2766 case NEON::BI__builtin_neon_vsraq_n_v:
2767 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
2768 Ops[1] = EmitNeonRShiftImm(Vec: Ops[1], Shift: Ops[2], Ty, usgn, name: "vsra_n");
2769 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
2770 case NEON::BI__builtin_neon_vst1q_lane_v:
2771 // Handle 64-bit integer elements as a special case. Use a shuffle to get
2772 // a one-element vector and avoid poor code for i64 in the backend.
2773 if (VTy->getElementType()->isIntegerTy(BitWidth: 64)) {
2774 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2775 Value *SV = llvm::ConstantVector::get(V: cast<llvm::Constant>(Val: Ops[2]));
2776 Ops[1] = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[1], Mask: SV);
2777 Ops[2] = getAlignmentValue32(PtrOp0);
2778 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
2779 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vst1,
2780 Tys), Args: Ops);
2781 }
2782 [[fallthrough]];
2783 case NEON::BI__builtin_neon_vst1_lane_v: {
2784 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2785 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2]);
2786 return Builder.CreateStore(Val: Ops[1],
2787 Addr: PtrOp0.withElementType(ElemTy: Ops[1]->getType()));
2788 }
2789 case NEON::BI__builtin_neon_vtbl1_v:
2790 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl1),
2791 Ops, name: "vtbl1");
2792 case NEON::BI__builtin_neon_vtbl2_v:
2793 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl2),
2794 Ops, name: "vtbl2");
2795 case NEON::BI__builtin_neon_vtbl3_v:
2796 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl3),
2797 Ops, name: "vtbl3");
2798 case NEON::BI__builtin_neon_vtbl4_v:
2799 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl4),
2800 Ops, name: "vtbl4");
2801 case NEON::BI__builtin_neon_vtbx1_v:
2802 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx1),
2803 Ops, name: "vtbx1");
2804 case NEON::BI__builtin_neon_vtbx2_v:
2805 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx2),
2806 Ops, name: "vtbx2");
2807 case NEON::BI__builtin_neon_vtbx3_v:
2808 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx3),
2809 Ops, name: "vtbx3");
2810 case NEON::BI__builtin_neon_vtbx4_v:
2811 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx4),
2812 Ops, name: "vtbx4");
2813 }
2814}
2815
2816template<typename Integer>
2817static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context) {
2818 return E->getIntegerConstantExpr(Ctx: Context)->getExtValue();
2819}
2820
2821static llvm::Value *SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V,
2822 llvm::Type *T, bool Unsigned) {
2823 // Helper function called by Tablegen-constructed ARM MVE builtin codegen,
2824 // which finds it convenient to specify signed/unsigned as a boolean flag.
2825 return Unsigned ? Builder.CreateZExt(V, DestTy: T) : Builder.CreateSExt(V, DestTy: T);
2826}
2827
2828static llvm::Value *MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V,
2829 uint32_t Shift, bool Unsigned) {
2830 // MVE helper function for integer shift right. This must handle signed vs
2831 // unsigned, and also deal specially with the case where the shift count is
2832 // equal to the lane size. In LLVM IR, an LShr with that parameter would be
2833 // undefined behavior, but in MVE it's legal, so we must convert it to code
2834 // that is not undefined in IR.
2835 unsigned LaneBits = cast<llvm::VectorType>(Val: V->getType())
2836 ->getElementType()
2837 ->getPrimitiveSizeInBits();
2838 if (Shift == LaneBits) {
2839 // An unsigned shift of the full lane size always generates zero, so we can
2840 // simply emit a zero vector. A signed shift of the full lane size does the
2841 // same thing as shifting by one bit fewer.
2842 if (Unsigned)
2843 return llvm::Constant::getNullValue(Ty: V->getType());
2844 else
2845 --Shift;
2846 }
2847 return Unsigned ? Builder.CreateLShr(LHS: V, RHS: Shift) : Builder.CreateAShr(LHS: V, RHS: Shift);
2848}
2849
2850static llvm::Value *ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V) {
2851 // MVE-specific helper function for a vector splat, which infers the element
2852 // count of the output vector by knowing that MVE vectors are all 128 bits
2853 // wide.
2854 unsigned Elements = 128 / V->getType()->getPrimitiveSizeInBits();
2855 return Builder.CreateVectorSplat(NumElts: Elements, V);
2856}
2857
2858static llvm::Value *ARMMVEVectorReinterpret(CGBuilderTy &Builder,
2859 CodeGenFunction *CGF,
2860 llvm::Value *V,
2861 llvm::Type *DestType) {
2862 // Convert one MVE vector type into another by reinterpreting its in-register
2863 // format.
2864 //
2865 // Little-endian, this is identical to a bitcast (which reinterprets the
2866 // memory format). But big-endian, they're not necessarily the same, because
2867 // the register and memory formats map to each other differently depending on
2868 // the lane size.
2869 //
2870 // We generate a bitcast whenever we can (if we're little-endian, or if the
2871 // lane sizes are the same anyway). Otherwise we fall back to an IR intrinsic
2872 // that performs the different kind of reinterpretation.
2873 if (CGF->getTarget().isBigEndian() &&
2874 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
2875 return Builder.CreateCall(
2876 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vreinterpretq,
2877 Tys: {DestType, V->getType()}),
2878 Args: V);
2879 } else {
2880 return Builder.CreateBitCast(V, DestTy: DestType);
2881 }
2882}
2883
2884static llvm::Value *VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd) {
2885 // Make a shufflevector that extracts every other element of a vector (evens
2886 // or odds, as desired).
2887 SmallVector<int, 16> Indices;
2888 unsigned InputElements =
2889 cast<llvm::FixedVectorType>(Val: V->getType())->getNumElements();
2890 for (unsigned i = 0; i < InputElements; i += 2)
2891 Indices.push_back(Elt: i + Odd);
2892 return Builder.CreateShuffleVector(V, Mask: Indices);
2893}
2894
2895static llvm::Value *VectorZip(CGBuilderTy &Builder, llvm::Value *V0,
2896 llvm::Value *V1) {
2897 // Make a shufflevector that interleaves two vectors element by element.
2898 assert(V0->getType() == V1->getType() && "Can't zip different vector types");
2899 SmallVector<int, 16> Indices;
2900 unsigned InputElements =
2901 cast<llvm::FixedVectorType>(Val: V0->getType())->getNumElements();
2902 for (unsigned i = 0; i < InputElements; i++) {
2903 Indices.push_back(Elt: i);
2904 Indices.push_back(Elt: i + InputElements);
2905 }
2906 return Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: Indices);
2907}
2908
2909template<unsigned HighBit, unsigned OtherBits>
2910static llvm::Value *ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT) {
2911 // MVE-specific helper function to make a vector splat of a constant such as
2912 // UINT_MAX or INT_MIN, in which all bits below the highest one are equal.
2913 llvm::Type *T = cast<llvm::VectorType>(Val: VT)->getElementType();
2914 unsigned LaneBits = T->getPrimitiveSizeInBits();
2915 uint32_t Value = HighBit << (LaneBits - 1);
2916 if (OtherBits)
2917 Value |= (1UL << (LaneBits - 1)) - 1;
2918 llvm::Value *Lane = llvm::ConstantInt::get(Ty: T, V: Value);
2919 return ARMMVEVectorSplat(Builder, V: Lane);
2920}
2921
2922static llvm::Value *ARMMVEVectorElementReverse(CGBuilderTy &Builder,
2923 llvm::Value *V,
2924 unsigned ReverseWidth) {
2925 // MVE-specific helper function which reverses the elements of a
2926 // vector within every (ReverseWidth)-bit collection of lanes.
2927 SmallVector<int, 16> Indices;
2928 unsigned LaneSize = V->getType()->getScalarSizeInBits();
2929 unsigned Elements = 128 / LaneSize;
2930 unsigned Mask = ReverseWidth / LaneSize - 1;
2931 for (unsigned i = 0; i < Elements; i++)
2932 Indices.push_back(Elt: i ^ Mask);
2933 return Builder.CreateShuffleVector(V, Mask: Indices);
2934}
2935
2936static llvm::Value *ARMMVECreateSIToFP(CGBuilderTy &Builder,
2937 CodeGenFunction *CGF, llvm::Value *V,
2938 llvm::Type *Ty) {
2939 return Builder.CreateCall(
2940 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_fp_int, Tys: {Ty, V->getType()}),
2941 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)});
2942}
2943
2944static llvm::Value *ARMMVECreateUIToFP(CGBuilderTy &Builder,
2945 CodeGenFunction *CGF, llvm::Value *V,
2946 llvm::Type *Ty) {
2947 return Builder.CreateCall(
2948 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_fp_int, Tys: {Ty, V->getType()}),
2949 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1)});
2950}
2951
2952static llvm::Value *ARMMVECreateFPToSI(CGBuilderTy &Builder,
2953 CodeGenFunction *CGF, llvm::Value *V,
2954 llvm::Type *Ty) {
2955 return Builder.CreateCall(
2956 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_int_fp, Tys: {Ty, V->getType()}),
2957 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)});
2958}
2959
2960static llvm::Value *ARMMVECreateFPToUI(CGBuilderTy &Builder,
2961 CodeGenFunction *CGF, llvm::Value *V,
2962 llvm::Type *Ty) {
2963 return Builder.CreateCall(
2964 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_int_fp, Tys: {Ty, V->getType()}),
2965 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1)});
2966}
2967
2968Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID,
2969 const CallExpr *E,
2970 ReturnValueSlot ReturnValue,
2971 llvm::Triple::ArchType Arch) {
2972 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
2973 Intrinsic::ID IRIntr;
2974 unsigned NumVectors;
2975
2976 // Code autogenerated by Tablegen will handle all the simple builtins.
2977 switch (BuiltinID) {
2978 #include "clang/Basic/arm_mve_builtin_cg.inc"
2979
2980 // If we didn't match an MVE builtin id at all, go back to the
2981 // main EmitARMBuiltinExpr.
2982 default:
2983 return nullptr;
2984 }
2985
2986 // Anything that breaks from that switch is an MVE builtin that
2987 // needs handwritten code to generate.
2988
2989 switch (CustomCodeGenType) {
2990
2991 case CustomCodeGen::VLD24: {
2992 llvm::SmallVector<Value *, 4> Ops;
2993 llvm::SmallVector<llvm::Type *, 4> Tys;
2994
2995 auto MvecCType = E->getType();
2996 auto MvecLType = ConvertType(T: MvecCType);
2997 assert(MvecLType->isStructTy() &&
2998 "Return type for vld[24]q should be a struct");
2999 assert(MvecLType->getStructNumElements() == 1 &&
3000 "Return-type struct for vld[24]q should have one element");
3001 auto MvecLTypeInner = MvecLType->getStructElementType(N: 0);
3002 assert(MvecLTypeInner->isArrayTy() &&
3003 "Return-type struct for vld[24]q should contain an array");
3004 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3005 "Array member of return-type struct vld[24]q has wrong length");
3006 auto VecLType = MvecLTypeInner->getArrayElementType();
3007
3008 Tys.push_back(Elt: VecLType);
3009
3010 auto Addr = E->getArg(Arg: 0);
3011 Ops.push_back(Elt: EmitScalarExpr(E: Addr));
3012 Tys.push_back(Elt: ConvertType(T: Addr->getType()));
3013
3014 Function *F = CGM.getIntrinsic(IID: IRIntr, Tys: ArrayRef(Tys));
3015 Value *LoadResult = Builder.CreateCall(Callee: F, Args: Ops);
3016 Value *MvecOut = PoisonValue::get(T: MvecLType);
3017 for (unsigned i = 0; i < NumVectors; ++i) {
3018 Value *Vec = Builder.CreateExtractValue(Agg: LoadResult, Idxs: i);
3019 MvecOut = Builder.CreateInsertValue(Agg: MvecOut, Val: Vec, Idxs: {0, i});
3020 }
3021
3022 if (ReturnValue.isNull())
3023 return MvecOut;
3024 else
3025 return Builder.CreateStore(Val: MvecOut, Addr: ReturnValue.getAddress());
3026 }
3027
3028 case CustomCodeGen::VST24: {
3029 llvm::SmallVector<Value *, 4> Ops;
3030 llvm::SmallVector<llvm::Type *, 4> Tys;
3031
3032 auto Addr = E->getArg(Arg: 0);
3033 Ops.push_back(Elt: EmitScalarExpr(E: Addr));
3034 Tys.push_back(Elt: ConvertType(T: Addr->getType()));
3035
3036 auto MvecCType = E->getArg(Arg: 1)->getType();
3037 auto MvecLType = ConvertType(T: MvecCType);
3038 assert(MvecLType->isStructTy() && "Data type for vst2q should be a struct");
3039 assert(MvecLType->getStructNumElements() == 1 &&
3040 "Data-type struct for vst2q should have one element");
3041 auto MvecLTypeInner = MvecLType->getStructElementType(N: 0);
3042 assert(MvecLTypeInner->isArrayTy() &&
3043 "Data-type struct for vst2q should contain an array");
3044 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3045 "Array member of return-type struct vld[24]q has wrong length");
3046 auto VecLType = MvecLTypeInner->getArrayElementType();
3047
3048 Tys.push_back(Elt: VecLType);
3049
3050 AggValueSlot MvecSlot = CreateAggTemp(T: MvecCType);
3051 EmitAggExpr(E: E->getArg(Arg: 1), AS: MvecSlot);
3052 auto Mvec = Builder.CreateLoad(Addr: MvecSlot.getAddress());
3053 for (unsigned i = 0; i < NumVectors; i++)
3054 Ops.push_back(Elt: Builder.CreateExtractValue(Agg: Mvec, Idxs: {0, i}));
3055
3056 Function *F = CGM.getIntrinsic(IID: IRIntr, Tys: ArrayRef(Tys));
3057 Value *ToReturn = nullptr;
3058 for (unsigned i = 0; i < NumVectors; i++) {
3059 Ops.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: i));
3060 ToReturn = Builder.CreateCall(Callee: F, Args: Ops);
3061 Ops.pop_back();
3062 }
3063 return ToReturn;
3064 }
3065 }
3066 llvm_unreachable("unknown custom codegen type.");
3067}
3068
3069Value *CodeGenFunction::EmitARMCDEBuiltinExpr(unsigned BuiltinID,
3070 const CallExpr *E,
3071 ReturnValueSlot ReturnValue,
3072 llvm::Triple::ArchType Arch) {
3073 switch (BuiltinID) {
3074 default:
3075 return nullptr;
3076#include "clang/Basic/arm_cde_builtin_cg.inc"
3077 }
3078}
3079
3080static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
3081 const CallExpr *E,
3082 SmallVectorImpl<Value *> &Ops,
3083 llvm::Triple::ArchType Arch) {
3084 unsigned int Int = 0;
3085 const char *s = nullptr;
3086
3087 switch (BuiltinID) {
3088 default:
3089 return nullptr;
3090 case NEON::BI__builtin_neon_vtbl1_v:
3091 case NEON::BI__builtin_neon_vqtbl1_v:
3092 case NEON::BI__builtin_neon_vqtbl1q_v:
3093 case NEON::BI__builtin_neon_vtbl2_v:
3094 case NEON::BI__builtin_neon_vqtbl2_v:
3095 case NEON::BI__builtin_neon_vqtbl2q_v:
3096 case NEON::BI__builtin_neon_vtbl3_v:
3097 case NEON::BI__builtin_neon_vqtbl3_v:
3098 case NEON::BI__builtin_neon_vqtbl3q_v:
3099 case NEON::BI__builtin_neon_vtbl4_v:
3100 case NEON::BI__builtin_neon_vqtbl4_v:
3101 case NEON::BI__builtin_neon_vqtbl4q_v:
3102 break;
3103 case NEON::BI__builtin_neon_vtbx1_v:
3104 case NEON::BI__builtin_neon_vqtbx1_v:
3105 case NEON::BI__builtin_neon_vqtbx1q_v:
3106 case NEON::BI__builtin_neon_vtbx2_v:
3107 case NEON::BI__builtin_neon_vqtbx2_v:
3108 case NEON::BI__builtin_neon_vqtbx2q_v:
3109 case NEON::BI__builtin_neon_vtbx3_v:
3110 case NEON::BI__builtin_neon_vqtbx3_v:
3111 case NEON::BI__builtin_neon_vqtbx3q_v:
3112 case NEON::BI__builtin_neon_vtbx4_v:
3113 case NEON::BI__builtin_neon_vqtbx4_v:
3114 case NEON::BI__builtin_neon_vqtbx4q_v:
3115 break;
3116 }
3117
3118 assert(E->getNumArgs() >= 3);
3119
3120 // Get the last argument, which specifies the vector type.
3121 const Expr *Arg = E->getArg(Arg: E->getNumArgs() - 1);
3122 std::optional<llvm::APSInt> Result =
3123 Arg->getIntegerConstantExpr(Ctx: CGF.getContext());
3124 if (!Result)
3125 return nullptr;
3126
3127 // Determine the type of this overloaded NEON intrinsic.
3128 NeonTypeFlags Type = Result->getZExtValue();
3129 llvm::FixedVectorType *Ty = GetNeonType(CGF: &CGF, TypeFlags: Type);
3130 if (!Ty)
3131 return nullptr;
3132
3133 CodeGen::CGBuilderTy &Builder = CGF.Builder;
3134
3135 // AArch64 scalar builtins are not overloaded, they do not have an extra
3136 // argument that specifies the vector type, need to handle each case.
3137 switch (BuiltinID) {
3138 case NEON::BI__builtin_neon_vtbl1_v: {
3139 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 1), ExtOp: nullptr, IndexOp: Ops[1],
3140 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl1, Name: "vtbl1");
3141 }
3142 case NEON::BI__builtin_neon_vtbl2_v: {
3143 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 2), ExtOp: nullptr, IndexOp: Ops[2],
3144 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl1, Name: "vtbl1");
3145 }
3146 case NEON::BI__builtin_neon_vtbl3_v: {
3147 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 3), ExtOp: nullptr, IndexOp: Ops[3],
3148 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl2, Name: "vtbl2");
3149 }
3150 case NEON::BI__builtin_neon_vtbl4_v: {
3151 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 4), ExtOp: nullptr, IndexOp: Ops[4],
3152 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl2, Name: "vtbl2");
3153 }
3154 case NEON::BI__builtin_neon_vtbx1_v: {
3155 Value *TblRes =
3156 packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 1), ExtOp: nullptr, IndexOp: Ops[2], ResTy: Ty,
3157 IntID: Intrinsic::aarch64_neon_tbl1, Name: "vtbl1");
3158
3159 llvm::Constant *EightV = ConstantInt::get(Ty, V: 8);
3160 Value *CmpRes = Builder.CreateICmp(P: ICmpInst::ICMP_UGE, LHS: Ops[2], RHS: EightV);
3161 CmpRes = Builder.CreateSExt(V: CmpRes, DestTy: Ty);
3162
3163 Value *EltsFromInput = Builder.CreateAnd(LHS: CmpRes, RHS: Ops[0]);
3164 Value *EltsFromTbl = Builder.CreateAnd(LHS: Builder.CreateNot(V: CmpRes), RHS: TblRes);
3165 return Builder.CreateOr(LHS: EltsFromInput, RHS: EltsFromTbl, Name: "vtbx");
3166 }
3167 case NEON::BI__builtin_neon_vtbx2_v: {
3168 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 2), ExtOp: Ops[0], IndexOp: Ops[3],
3169 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbx1, Name: "vtbx1");
3170 }
3171 case NEON::BI__builtin_neon_vtbx3_v: {
3172 Value *TblRes =
3173 packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 3), ExtOp: nullptr, IndexOp: Ops[4], ResTy: Ty,
3174 IntID: Intrinsic::aarch64_neon_tbl2, Name: "vtbl2");
3175
3176 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, V: 24);
3177 Value *CmpRes = Builder.CreateICmp(P: ICmpInst::ICMP_UGE, LHS: Ops[4],
3178 RHS: TwentyFourV);
3179 CmpRes = Builder.CreateSExt(V: CmpRes, DestTy: Ty);
3180
3181 Value *EltsFromInput = Builder.CreateAnd(LHS: CmpRes, RHS: Ops[0]);
3182 Value *EltsFromTbl = Builder.CreateAnd(LHS: Builder.CreateNot(V: CmpRes), RHS: TblRes);
3183 return Builder.CreateOr(LHS: EltsFromInput, RHS: EltsFromTbl, Name: "vtbx");
3184 }
3185 case NEON::BI__builtin_neon_vtbx4_v: {
3186 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 4), ExtOp: Ops[0], IndexOp: Ops[5],
3187 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbx2, Name: "vtbx2");
3188 }
3189 case NEON::BI__builtin_neon_vqtbl1_v:
3190 case NEON::BI__builtin_neon_vqtbl1q_v:
3191 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
3192 case NEON::BI__builtin_neon_vqtbl2_v:
3193 case NEON::BI__builtin_neon_vqtbl2q_v: {
3194 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
3195 case NEON::BI__builtin_neon_vqtbl3_v:
3196 case NEON::BI__builtin_neon_vqtbl3q_v:
3197 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
3198 case NEON::BI__builtin_neon_vqtbl4_v:
3199 case NEON::BI__builtin_neon_vqtbl4q_v:
3200 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
3201 case NEON::BI__builtin_neon_vqtbx1_v:
3202 case NEON::BI__builtin_neon_vqtbx1q_v:
3203 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
3204 case NEON::BI__builtin_neon_vqtbx2_v:
3205 case NEON::BI__builtin_neon_vqtbx2q_v:
3206 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
3207 case NEON::BI__builtin_neon_vqtbx3_v:
3208 case NEON::BI__builtin_neon_vqtbx3q_v:
3209 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
3210 case NEON::BI__builtin_neon_vqtbx4_v:
3211 case NEON::BI__builtin_neon_vqtbx4q_v:
3212 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
3213 }
3214 }
3215
3216 if (!Int)
3217 return nullptr;
3218
3219 Function *F = CGF.CGM.getIntrinsic(IID: Int, Tys: Ty);
3220 return CGF.EmitNeonCall(F, Ops, name: s);
3221}
3222
3223Value *CodeGenFunction::vectorWrapScalar16(Value *Op) {
3224 auto *VTy = llvm::FixedVectorType::get(ElementType: Int16Ty, NumElts: 4);
3225 Op = Builder.CreateBitCast(V: Op, DestTy: Int16Ty);
3226 Value *V = PoisonValue::get(T: VTy);
3227 llvm::Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
3228 Op = Builder.CreateInsertElement(Vec: V, NewElt: Op, Idx: CI);
3229 return Op;
3230}
3231
3232/// SVEBuiltinMemEltTy - Returns the memory element type for this memory
3233/// access builtin. Only required if it can't be inferred from the base pointer
3234/// operand.
3235llvm::Type *CodeGenFunction::SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags) {
3236 switch (TypeFlags.getMemEltType()) {
3237 case SVETypeFlags::MemEltTyDefault:
3238 return getEltType(TypeFlags);
3239 case SVETypeFlags::MemEltTyInt8:
3240 return Builder.getInt8Ty();
3241 case SVETypeFlags::MemEltTyInt16:
3242 return Builder.getInt16Ty();
3243 case SVETypeFlags::MemEltTyInt32:
3244 return Builder.getInt32Ty();
3245 case SVETypeFlags::MemEltTyInt64:
3246 return Builder.getInt64Ty();
3247 }
3248 llvm_unreachable("Unknown MemEltType");
3249}
3250
3251llvm::Type *CodeGenFunction::getEltType(const SVETypeFlags &TypeFlags) {
3252 switch (TypeFlags.getEltType()) {
3253 default:
3254 llvm_unreachable("Invalid SVETypeFlag!");
3255
3256 case SVETypeFlags::EltTyMFloat8:
3257 case SVETypeFlags::EltTyInt8:
3258 return Builder.getInt8Ty();
3259 case SVETypeFlags::EltTyInt16:
3260 return Builder.getInt16Ty();
3261 case SVETypeFlags::EltTyInt32:
3262 return Builder.getInt32Ty();
3263 case SVETypeFlags::EltTyInt64:
3264 return Builder.getInt64Ty();
3265 case SVETypeFlags::EltTyInt128:
3266 return Builder.getInt128Ty();
3267
3268 case SVETypeFlags::EltTyFloat16:
3269 return Builder.getHalfTy();
3270 case SVETypeFlags::EltTyFloat32:
3271 return Builder.getFloatTy();
3272 case SVETypeFlags::EltTyFloat64:
3273 return Builder.getDoubleTy();
3274
3275 case SVETypeFlags::EltTyBFloat16:
3276 return Builder.getBFloatTy();
3277
3278 case SVETypeFlags::EltTyBool8:
3279 case SVETypeFlags::EltTyBool16:
3280 case SVETypeFlags::EltTyBool32:
3281 case SVETypeFlags::EltTyBool64:
3282 return Builder.getInt1Ty();
3283 }
3284}
3285
3286// Return the llvm predicate vector type corresponding to the specified element
3287// TypeFlags.
3288llvm::ScalableVectorType *
3289CodeGenFunction::getSVEPredType(const SVETypeFlags &TypeFlags) {
3290 switch (TypeFlags.getEltType()) {
3291 default: llvm_unreachable("Unhandled SVETypeFlag!");
3292
3293 case SVETypeFlags::EltTyInt8:
3294 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 16);
3295 case SVETypeFlags::EltTyInt16:
3296 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3297 case SVETypeFlags::EltTyInt32:
3298 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3299 case SVETypeFlags::EltTyInt64:
3300 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3301
3302 case SVETypeFlags::EltTyBFloat16:
3303 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3304 case SVETypeFlags::EltTyFloat16:
3305 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3306 case SVETypeFlags::EltTyFloat32:
3307 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3308 case SVETypeFlags::EltTyFloat64:
3309 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3310
3311 case SVETypeFlags::EltTyBool8:
3312 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 16);
3313 case SVETypeFlags::EltTyBool16:
3314 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3315 case SVETypeFlags::EltTyBool32:
3316 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3317 case SVETypeFlags::EltTyBool64:
3318 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3319 }
3320}
3321
3322// Return the llvm vector type corresponding to the specified element TypeFlags.
3323llvm::ScalableVectorType *
3324CodeGenFunction::getSVEType(const SVETypeFlags &TypeFlags) {
3325 switch (TypeFlags.getEltType()) {
3326 default:
3327 llvm_unreachable("Invalid SVETypeFlag!");
3328
3329 case SVETypeFlags::EltTyInt8:
3330 return llvm::ScalableVectorType::get(ElementType: Builder.getInt8Ty(), MinNumElts: 16);
3331 case SVETypeFlags::EltTyInt16:
3332 return llvm::ScalableVectorType::get(ElementType: Builder.getInt16Ty(), MinNumElts: 8);
3333 case SVETypeFlags::EltTyInt32:
3334 return llvm::ScalableVectorType::get(ElementType: Builder.getInt32Ty(), MinNumElts: 4);
3335 case SVETypeFlags::EltTyInt64:
3336 return llvm::ScalableVectorType::get(ElementType: Builder.getInt64Ty(), MinNumElts: 2);
3337
3338 case SVETypeFlags::EltTyMFloat8:
3339 return llvm::ScalableVectorType::get(ElementType: Builder.getInt8Ty(), MinNumElts: 16);
3340 case SVETypeFlags::EltTyFloat16:
3341 return llvm::ScalableVectorType::get(ElementType: Builder.getHalfTy(), MinNumElts: 8);
3342 case SVETypeFlags::EltTyBFloat16:
3343 return llvm::ScalableVectorType::get(ElementType: Builder.getBFloatTy(), MinNumElts: 8);
3344 case SVETypeFlags::EltTyFloat32:
3345 return llvm::ScalableVectorType::get(ElementType: Builder.getFloatTy(), MinNumElts: 4);
3346 case SVETypeFlags::EltTyFloat64:
3347 return llvm::ScalableVectorType::get(ElementType: Builder.getDoubleTy(), MinNumElts: 2);
3348
3349 case SVETypeFlags::EltTyBool8:
3350 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 16);
3351 case SVETypeFlags::EltTyBool16:
3352 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3353 case SVETypeFlags::EltTyBool32:
3354 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3355 case SVETypeFlags::EltTyBool64:
3356 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3357 }
3358}
3359
3360constexpr unsigned SVEBitsPerBlock = 128;
3361
3362static llvm::ScalableVectorType *getSVEVectorForElementType(llvm::Type *EltTy) {
3363 unsigned NumElts = SVEBitsPerBlock / EltTy->getScalarSizeInBits();
3364 return llvm::ScalableVectorType::get(ElementType: EltTy, MinNumElts: NumElts);
3365}
3366
3367// Reinterpret the input predicate so that it can be used to correctly isolate
3368// the elements of the specified datatype.
3369Value *CodeGenFunction::EmitSVEPredicateCast(Value *Pred,
3370 llvm::ScalableVectorType *VTy) {
3371
3372 if (isa<TargetExtType>(Val: Pred->getType()) &&
3373 cast<TargetExtType>(Val: Pred->getType())->getName() == "aarch64.svcount")
3374 return Pred;
3375
3376 auto *RTy = llvm::VectorType::get(ElementType: IntegerType::get(C&: getLLVMContext(), NumBits: 1), Other: VTy);
3377 if (Pred->getType() == RTy)
3378 return Pred;
3379
3380 unsigned IntID;
3381 llvm::Type *IntrinsicTy;
3382 switch (VTy->getMinNumElements()) {
3383 default:
3384 llvm_unreachable("unsupported element count!");
3385 case 1:
3386 case 2:
3387 case 4:
3388 case 8:
3389 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
3390 IntrinsicTy = RTy;
3391 break;
3392 case 16:
3393 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
3394 IntrinsicTy = Pred->getType();
3395 break;
3396 }
3397
3398 Function *F = CGM.getIntrinsic(IID: IntID, Tys: IntrinsicTy);
3399 Value *C = Builder.CreateCall(Callee: F, Args: Pred);
3400 assert(C->getType() == RTy && "Unexpected return type!");
3401 return C;
3402}
3403
3404Value *CodeGenFunction::EmitSVEPredicateTupleCast(Value *PredTuple,
3405 llvm::StructType *Ty) {
3406 if (PredTuple->getType() == Ty)
3407 return PredTuple;
3408
3409 Value *Ret = llvm::PoisonValue::get(T: Ty);
3410 for (unsigned I = 0; I < Ty->getNumElements(); ++I) {
3411 Value *Pred = Builder.CreateExtractValue(Agg: PredTuple, Idxs: I);
3412 Pred = EmitSVEPredicateCast(
3413 Pred, VTy: cast<llvm::ScalableVectorType>(Val: Ty->getTypeAtIndex(N: I)));
3414 Ret = Builder.CreateInsertValue(Agg: Ret, Val: Pred, Idxs: I);
3415 }
3416
3417 return Ret;
3418}
3419
3420Value *CodeGenFunction::EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
3421 SmallVectorImpl<Value *> &Ops,
3422 unsigned IntID) {
3423 auto *ResultTy = getSVEType(TypeFlags);
3424 auto *OverloadedTy =
3425 llvm::ScalableVectorType::get(ElementType: SVEBuiltinMemEltTy(TypeFlags), SVTy: ResultTy);
3426 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {OverloadedTy, Ops[1]->getType()});
3427
3428 // At the ACLE level there's only one predicate type, svbool_t, which is
3429 // mapped to <n x 16 x i1>. However, this might be incompatible with the
3430 // actual type being loaded. For example, when loading doubles (i64) the
3431 // predicate should be <n x 2 x i1> instead. At the IR level the type of
3432 // the predicate and the data being loaded must match. Cast to the type
3433 // expected by the intrinsic. The intrinsic itself should be defined in
3434 // a way than enforces relations between parameter types.
3435 Ops[0] = EmitSVEPredicateCast(
3436 Pred: Ops[0], VTy: cast<llvm::ScalableVectorType>(Val: F->getArg(i: 0)->getType()));
3437
3438 // Pass 0 when the offset is missing. This can only be applied when using
3439 // the "vector base" addressing mode for which ACLE allows no offset. The
3440 // corresponding LLVM IR always requires an offset.
3441 if (Ops.size() == 2) {
3442 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
3443 Ops.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: 0));
3444 }
3445
3446 // For "vector base, scalar index" scale the index so that it becomes a
3447 // scalar offset.
3448 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
3449 unsigned BytesPerElt =
3450 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
3451 Ops[2] = Builder.CreateShl(LHS: Ops[2], RHS: Log2_32(Value: BytesPerElt));
3452 }
3453
3454 Value *Call = Builder.CreateCall(Callee: F, Args: Ops);
3455
3456 // The following sext/zext is only needed when ResultTy != OverloadedTy. In
3457 // other cases it's folded into a nop.
3458 return TypeFlags.isZExtReturn() ? Builder.CreateZExt(V: Call, DestTy: ResultTy)
3459 : Builder.CreateSExt(V: Call, DestTy: ResultTy);
3460}
3461
3462Value *CodeGenFunction::EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
3463 SmallVectorImpl<Value *> &Ops,
3464 unsigned IntID) {
3465 auto *SrcDataTy = getSVEType(TypeFlags);
3466 auto *OverloadedTy =
3467 llvm::ScalableVectorType::get(ElementType: SVEBuiltinMemEltTy(TypeFlags), SVTy: SrcDataTy);
3468
3469 // In ACLE the source data is passed in the last argument, whereas in LLVM IR
3470 // it's the first argument. Move it accordingly.
3471 Ops.insert(I: Ops.begin(), Elt: Ops.pop_back_val());
3472
3473 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {OverloadedTy, Ops[2]->getType()});
3474
3475 // Pass 0 when the offset is missing. This can only be applied when using
3476 // the "vector base" addressing mode for which ACLE allows no offset. The
3477 // corresponding LLVM IR always requires an offset.
3478 if (Ops.size() == 3) {
3479 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
3480 Ops.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: 0));
3481 }
3482
3483 // Truncation is needed when SrcDataTy != OverloadedTy. In other cases it's
3484 // folded into a nop.
3485 Ops[0] = Builder.CreateTrunc(V: Ops[0], DestTy: OverloadedTy);
3486
3487 // At the ACLE level there's only one predicate type, svbool_t, which is
3488 // mapped to <n x 16 x i1>. However, this might be incompatible with the
3489 // actual type being stored. For example, when storing doubles (i64) the
3490 // predicated should be <n x 2 x i1> instead. At the IR level the type of
3491 // the predicate and the data being stored must match. Cast to the type
3492 // expected by the intrinsic. The intrinsic itself should be defined in
3493 // a way that enforces relations between parameter types.
3494 Ops[1] = EmitSVEPredicateCast(
3495 Pred: Ops[1], VTy: cast<llvm::ScalableVectorType>(Val: F->getArg(i: 1)->getType()));
3496
3497 // For "vector base, scalar index" scale the index so that it becomes a
3498 // scalar offset.
3499 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
3500 unsigned BytesPerElt =
3501 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
3502 Ops[3] = Builder.CreateShl(LHS: Ops[3], RHS: Log2_32(Value: BytesPerElt));
3503 }
3504
3505 return Builder.CreateCall(Callee: F, Args: Ops);
3506}
3507
3508Value *CodeGenFunction::EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
3509 SmallVectorImpl<Value *> &Ops,
3510 unsigned IntID) {
3511 // The gather prefetches are overloaded on the vector input - this can either
3512 // be the vector of base addresses or vector of offsets.
3513 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Val: Ops[1]->getType());
3514 if (!OverloadedTy)
3515 OverloadedTy = cast<llvm::ScalableVectorType>(Val: Ops[2]->getType());
3516
3517 // Cast the predicate from svbool_t to the right number of elements.
3518 Ops[0] = EmitSVEPredicateCast(Pred: Ops[0], VTy: OverloadedTy);
3519
3520 // vector + imm addressing modes
3521 if (Ops[1]->getType()->isVectorTy()) {
3522 if (Ops.size() == 3) {
3523 // Pass 0 for 'vector+imm' when the index is omitted.
3524 Ops.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: 0));
3525
3526 // The sv_prfop is the last operand in the builtin and IR intrinsic.
3527 std::swap(a&: Ops[2], b&: Ops[3]);
3528 } else {
3529 // Index needs to be passed as scaled offset.
3530 llvm::Type *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
3531 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
3532 if (BytesPerElt > 1)
3533 Ops[2] = Builder.CreateShl(LHS: Ops[2], RHS: Log2_32(Value: BytesPerElt));
3534 }
3535
3536 Function *F = CGM.getIntrinsic(IID: IntID, Tys: OverloadedTy);
3537 return Builder.CreateCall(Callee: F, Args: Ops);
3538 }
3539
3540 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {Ops[1]->getType(), OverloadedTy});
3541 return Builder.CreateCall(Callee: F, Args: Ops);
3542}
3543
3544Value *CodeGenFunction::EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
3545 SmallVectorImpl<Value*> &Ops,
3546 unsigned IntID) {
3547 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
3548 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy);
3549 Value *BasePtr = Ops[1];
3550
3551 // Does the load have an offset?
3552 if (Ops.size() > 2)
3553 BasePtr = Builder.CreateGEP(Ty: VTy, Ptr: BasePtr, IdxList: Ops[2]);
3554
3555 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {VTy, BasePtr->getType()});
3556 return Builder.CreateCall(Callee: F, Args: {Predicate, BasePtr});
3557}
3558
3559Value *CodeGenFunction::EmitSVEStructStore(const SVETypeFlags &TypeFlags,
3560 SmallVectorImpl<Value*> &Ops,
3561 unsigned IntID) {
3562 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
3563
3564 unsigned N;
3565 switch (IntID) {
3566 case Intrinsic::aarch64_sve_st2:
3567 case Intrinsic::aarch64_sve_st1_pn_x2:
3568 case Intrinsic::aarch64_sve_stnt1_pn_x2:
3569 case Intrinsic::aarch64_sve_st2q:
3570 N = 2;
3571 break;
3572 case Intrinsic::aarch64_sve_st3:
3573 case Intrinsic::aarch64_sve_st3q:
3574 N = 3;
3575 break;
3576 case Intrinsic::aarch64_sve_st4:
3577 case Intrinsic::aarch64_sve_st1_pn_x4:
3578 case Intrinsic::aarch64_sve_stnt1_pn_x4:
3579 case Intrinsic::aarch64_sve_st4q:
3580 N = 4;
3581 break;
3582 default:
3583 llvm_unreachable("unknown intrinsic!");
3584 }
3585
3586 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy);
3587 Value *BasePtr = Ops[1];
3588
3589 // Does the store have an offset?
3590 if (Ops.size() > (2 + N))
3591 BasePtr = Builder.CreateGEP(Ty: VTy, Ptr: BasePtr, IdxList: Ops[2]);
3592
3593 // The llvm.aarch64.sve.st2/3/4 intrinsics take legal part vectors, so we
3594 // need to break up the tuple vector.
3595 SmallVector<llvm::Value*, 5> Operands;
3596 for (unsigned I = Ops.size() - N; I < Ops.size(); ++I)
3597 Operands.push_back(Elt: Ops[I]);
3598 Operands.append(IL: {Predicate, BasePtr});
3599 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {VTy, BasePtr->getType()});
3600
3601 return Builder.CreateCall(Callee: F, Args: Operands);
3602}
3603
3604// SVE2's svpmullb and svpmullt builtins are similar to the svpmullb_pair and
3605// svpmullt_pair intrinsics, with the exception that their results are bitcast
3606// to a wider type.
3607Value *CodeGenFunction::EmitSVEPMull(const SVETypeFlags &TypeFlags,
3608 SmallVectorImpl<Value *> &Ops,
3609 unsigned BuiltinID) {
3610 // Splat scalar operand to vector (intrinsics with _n infix)
3611 if (TypeFlags.hasSplatOperand()) {
3612 unsigned OpNo = TypeFlags.getSplatOperand();
3613 Ops[OpNo] = EmitSVEDupX(Scalar: Ops[OpNo]);
3614 }
3615
3616 // The pair-wise function has a narrower overloaded type.
3617 Function *F = CGM.getIntrinsic(IID: BuiltinID, Tys: Ops[0]->getType());
3618 Value *Call = Builder.CreateCall(Callee: F, Args: {Ops[0], Ops[1]});
3619
3620 // Now bitcast to the wider result type.
3621 llvm::ScalableVectorType *Ty = getSVEType(TypeFlags);
3622 return EmitSVEReinterpret(Val: Call, Ty);
3623}
3624
3625Value *CodeGenFunction::EmitSVEMovl(const SVETypeFlags &TypeFlags,
3626 ArrayRef<Value *> Ops, unsigned BuiltinID) {
3627 llvm::Type *OverloadedTy = getSVEType(TypeFlags);
3628 Function *F = CGM.getIntrinsic(IID: BuiltinID, Tys: OverloadedTy);
3629 return Builder.CreateCall(Callee: F, Args: {Ops[0], Builder.getInt32(C: 0)});
3630}
3631
3632Value *CodeGenFunction::EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
3633 SmallVectorImpl<Value *> &Ops,
3634 unsigned BuiltinID) {
3635 auto *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
3636 auto *VectorTy = getSVEVectorForElementType(EltTy: MemEltTy);
3637 auto *MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, SVTy: VectorTy);
3638
3639 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy: MemoryTy);
3640 Value *BasePtr = Ops[1];
3641
3642 // Implement the index operand if not omitted.
3643 if (Ops.size() > 3)
3644 BasePtr = Builder.CreateGEP(Ty: MemoryTy, Ptr: BasePtr, IdxList: Ops[2]);
3645
3646 Value *PrfOp = Ops.back();
3647
3648 llvm::Type *Tys[2] = {Predicate->getType(), BasePtr->getType()};
3649 Function *F = CGM.getIntrinsic(IID: BuiltinID, Tys);
3650 return Builder.CreateCall(Callee: F, Args: {Predicate, BasePtr, PrfOp});
3651}
3652
3653Value *CodeGenFunction::EmitSVEMaskedLoad(const CallExpr *E,
3654 llvm::Type *ReturnTy,
3655 SmallVectorImpl<Value *> &Ops,
3656 unsigned IntrinsicID,
3657 bool IsZExtReturn) {
3658 QualType LangPTy = E->getArg(Arg: 1)->getType();
3659 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
3660 T: LangPTy->castAs<PointerType>()->getPointeeType());
3661
3662 // Mfloat8 types is stored as a vector, so extra work
3663 // to extract sclar element type is necessary.
3664 if (MemEltTy->isVectorTy()) {
3665 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
3666 "Only <1 x i8> expected");
3667 MemEltTy = cast<llvm::VectorType>(Val: MemEltTy)->getElementType();
3668 }
3669
3670 // The vector type that is returned may be different from the
3671 // eventual type loaded from memory.
3672 auto VectorTy = cast<llvm::ScalableVectorType>(Val: ReturnTy);
3673 llvm::ScalableVectorType *MemoryTy = nullptr;
3674 llvm::ScalableVectorType *PredTy = nullptr;
3675 bool IsQuadLoad = false;
3676 switch (IntrinsicID) {
3677 case Intrinsic::aarch64_sve_ld1uwq:
3678 case Intrinsic::aarch64_sve_ld1udq:
3679 MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, MinNumElts: 1);
3680 PredTy = llvm::ScalableVectorType::get(
3681 ElementType: llvm::Type::getInt1Ty(C&: getLLVMContext()), MinNumElts: 1);
3682 IsQuadLoad = true;
3683 break;
3684 default:
3685 MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, SVTy: VectorTy);
3686 PredTy = MemoryTy;
3687 break;
3688 }
3689
3690 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy: PredTy);
3691 Value *BasePtr = Ops[1];
3692
3693 // Does the load have an offset?
3694 if (Ops.size() > 2)
3695 BasePtr = Builder.CreateGEP(Ty: MemoryTy, Ptr: BasePtr, IdxList: Ops[2]);
3696
3697 llvm::Type *Tys[2] = {IsQuadLoad ? VectorTy : MemoryTy, BasePtr->getType()};
3698 Function *F = CGM.getIntrinsic(IID: IntrinsicID, Tys);
3699 auto *Load = Builder.CreateCall(Callee: F, Args: {Predicate, BasePtr});
3700 auto TBAAInfo = CGM.getTBAAAccessInfo(AccessType: LangPTy->getPointeeType());
3701 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo);
3702
3703 if (IsQuadLoad)
3704 return Load;
3705
3706 return IsZExtReturn ? Builder.CreateZExt(V: Load, DestTy: VectorTy)
3707 : Builder.CreateSExt(V: Load, DestTy: VectorTy);
3708}
3709
3710Value *CodeGenFunction::EmitSVEMaskedStore(const CallExpr *E,
3711 SmallVectorImpl<Value *> &Ops,
3712 unsigned IntrinsicID) {
3713 QualType LangPTy = E->getArg(Arg: 1)->getType();
3714 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
3715 T: LangPTy->castAs<PointerType>()->getPointeeType());
3716
3717 // Mfloat8 types is stored as a vector, so extra work
3718 // to extract sclar element type is necessary.
3719 if (MemEltTy->isVectorTy()) {
3720 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
3721 "Only <1 x i8> expected");
3722 MemEltTy = cast<llvm::VectorType>(Val: MemEltTy)->getElementType();
3723 }
3724
3725 // The vector type that is stored may be different from the
3726 // eventual type stored to memory.
3727 auto VectorTy = cast<llvm::ScalableVectorType>(Val: Ops.back()->getType());
3728 auto MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, SVTy: VectorTy);
3729
3730 auto PredTy = MemoryTy;
3731 auto AddrMemoryTy = MemoryTy;
3732 bool IsQuadStore = false;
3733
3734 switch (IntrinsicID) {
3735 case Intrinsic::aarch64_sve_st1wq:
3736 case Intrinsic::aarch64_sve_st1dq:
3737 AddrMemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, MinNumElts: 1);
3738 PredTy =
3739 llvm::ScalableVectorType::get(ElementType: IntegerType::get(C&: getLLVMContext(), NumBits: 1), MinNumElts: 1);
3740 IsQuadStore = true;
3741 break;
3742 default:
3743 break;
3744 }
3745 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy: PredTy);
3746 Value *BasePtr = Ops[1];
3747
3748 // Does the store have an offset?
3749 if (Ops.size() == 4)
3750 BasePtr = Builder.CreateGEP(Ty: AddrMemoryTy, Ptr: BasePtr, IdxList: Ops[2]);
3751
3752 // Last value is always the data
3753 Value *Val =
3754 IsQuadStore ? Ops.back() : Builder.CreateTrunc(V: Ops.back(), DestTy: MemoryTy);
3755
3756 llvm::Type *Tys[2] = {IsQuadStore ? VectorTy : MemoryTy, BasePtr->getType()};
3757 Function *F = CGM.getIntrinsic(IID: IntrinsicID, Tys);
3758 auto *Store = Builder.CreateCall(Callee: F, Args: {Val, Predicate, BasePtr});
3759 auto TBAAInfo = CGM.getTBAAAccessInfo(AccessType: LangPTy->getPointeeType());
3760 CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
3761 return Store;
3762}
3763
3764Value *CodeGenFunction::EmitSMELd1St1(const SVETypeFlags &TypeFlags,
3765 SmallVectorImpl<Value *> &Ops,
3766 unsigned IntID) {
3767 Ops[2] = EmitSVEPredicateCast(
3768 Pred: Ops[2], VTy: getSVEVectorForElementType(EltTy: SVEBuiltinMemEltTy(TypeFlags)));
3769
3770 SmallVector<Value *> NewOps;
3771 NewOps.push_back(Elt: Ops[2]);
3772
3773 llvm::Value *BasePtr = Ops[3];
3774 llvm::Value *RealSlice = Ops[1];
3775 // If the intrinsic contains the vnum parameter, multiply it with the vector
3776 // size in bytes.
3777 if (Ops.size() == 5) {
3778 Function *StreamingVectorLength =
3779 CGM.getIntrinsic(IID: Intrinsic::aarch64_sme_cntsd);
3780 llvm::Value *StreamingVectorLengthCall =
3781 Builder.CreateMul(LHS: Builder.CreateCall(Callee: StreamingVectorLength),
3782 RHS: llvm::ConstantInt::get(Ty: Int64Ty, V: 8), Name: "svl",
3783 /* HasNUW */ true, /* HasNSW */ true);
3784 llvm::Value *Mulvl =
3785 Builder.CreateMul(LHS: StreamingVectorLengthCall, RHS: Ops[4], Name: "mulvl");
3786 // The type of the ptr parameter is void *, so use Int8Ty here.
3787 BasePtr = Builder.CreateGEP(Ty: Int8Ty, Ptr: Ops[3], IdxList: Mulvl);
3788 RealSlice = Builder.CreateZExt(V: RealSlice, DestTy: Int64Ty);
3789 RealSlice = Builder.CreateAdd(LHS: RealSlice, RHS: Ops[4]);
3790 RealSlice = Builder.CreateTrunc(V: RealSlice, DestTy: Int32Ty);
3791 }
3792 NewOps.push_back(Elt: BasePtr);
3793 NewOps.push_back(Elt: Ops[0]);
3794 NewOps.push_back(Elt: RealSlice);
3795 Function *F = CGM.getIntrinsic(IID: IntID, Tys: BasePtr->getType());
3796 return Builder.CreateCall(Callee: F, Args: NewOps);
3797}
3798
3799Value *CodeGenFunction::EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
3800 SmallVectorImpl<Value *> &Ops,
3801 unsigned IntID) {
3802 auto *VecTy = getSVEType(TypeFlags);
3803 Function *F = CGM.getIntrinsic(IID: IntID, Tys: VecTy);
3804 if (TypeFlags.isReadZA())
3805 Ops[1] = EmitSVEPredicateCast(Pred: Ops[1], VTy: VecTy);
3806 else if (TypeFlags.isWriteZA())
3807 Ops[2] = EmitSVEPredicateCast(Pred: Ops[2], VTy: VecTy);
3808 return Builder.CreateCall(Callee: F, Args: Ops);
3809}
3810
3811Value *CodeGenFunction::EmitSMEZero(const SVETypeFlags &TypeFlags,
3812 SmallVectorImpl<Value *> &Ops,
3813 unsigned IntID) {
3814 // svzero_za() intrinsic zeros the entire za tile and has no paramters.
3815 if (Ops.size() == 0)
3816 Ops.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: 255));
3817 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {});
3818 return Builder.CreateCall(Callee: F, Args: Ops);
3819}
3820
3821Value *CodeGenFunction::EmitSMELdrStr(const SVETypeFlags &TypeFlags,
3822 SmallVectorImpl<Value *> &Ops,
3823 unsigned IntID) {
3824 if (Ops.size() == 2)
3825 Ops.push_back(Elt: Builder.getInt32(C: 0));
3826 else
3827 Ops[2] = Builder.CreateIntCast(V: Ops[2], DestTy: Int32Ty, isSigned: true);
3828 Function *F = CGM.getIntrinsic(IID: IntID, Tys: Ops[1]->getType());
3829 return Builder.CreateCall(Callee: F, Args: Ops);
3830}
3831
3832// Limit the usage of scalable llvm IR generated by the ACLE by using the
3833// sve dup.x intrinsic instead of IRBuilder::CreateVectorSplat.
3834Value *CodeGenFunction::EmitSVEDupX(Value *Scalar, llvm::Type *Ty) {
3835 return Builder.CreateVectorSplat(
3836 EC: cast<llvm::VectorType>(Val: Ty)->getElementCount(), V: Scalar);
3837}
3838
3839Value *CodeGenFunction::EmitSVEDupX(Value *Scalar) {
3840 if (auto *Ty = Scalar->getType(); Ty->isVectorTy()) {
3841#ifndef NDEBUG
3842 auto *VecTy = cast<llvm::VectorType>(Ty);
3843 ElementCount EC = VecTy->getElementCount();
3844 assert(EC.isScalar() && VecTy->getElementType() == Int8Ty &&
3845 "Only <1 x i8> expected");
3846#endif
3847 Scalar = Builder.CreateExtractElement(Vec: Scalar, Idx: uint64_t(0));
3848 }
3849 return EmitSVEDupX(Scalar, Ty: getSVEVectorForElementType(EltTy: Scalar->getType()));
3850}
3851
3852Value *CodeGenFunction::EmitSVEReinterpret(Value *Val, llvm::Type *Ty) {
3853 // FIXME: For big endian this needs an additional REV, or needs a separate
3854 // intrinsic that is code-generated as a no-op, because the LLVM bitcast
3855 // instruction is defined as 'bitwise' equivalent from memory point of
3856 // view (when storing/reloading), whereas the svreinterpret builtin
3857 // implements bitwise equivalent cast from register point of view.
3858 // LLVM CodeGen for a bitcast must add an explicit REV for big-endian.
3859
3860 if (auto *StructTy = dyn_cast<StructType>(Val: Ty)) {
3861 Value *Tuple = llvm::PoisonValue::get(T: Ty);
3862
3863 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
3864 Value *In = Builder.CreateExtractValue(Agg: Val, Idxs: I);
3865 Value *Out = Builder.CreateBitCast(V: In, DestTy: StructTy->getTypeAtIndex(N: I));
3866 Tuple = Builder.CreateInsertValue(Agg: Tuple, Val: Out, Idxs: I);
3867 }
3868
3869 return Tuple;
3870 }
3871
3872 return Builder.CreateBitCast(V: Val, DestTy: Ty);
3873}
3874
3875static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty,
3876 SmallVectorImpl<Value *> &Ops) {
3877 auto *SplatZero = Constant::getNullValue(Ty);
3878 Ops.insert(I: Ops.begin(), Elt: SplatZero);
3879}
3880
3881static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty,
3882 SmallVectorImpl<Value *> &Ops) {
3883 auto *SplatUndef = UndefValue::get(T: Ty);
3884 Ops.insert(I: Ops.begin(), Elt: SplatUndef);
3885}
3886
3887SmallVector<llvm::Type *, 2>
3888CodeGenFunction::getSVEOverloadTypes(const SVETypeFlags &TypeFlags,
3889 llvm::Type *ResultType,
3890 ArrayRef<Value *> Ops) {
3891 if (TypeFlags.isOverloadNone())
3892 return {};
3893
3894 llvm::Type *DefaultType = getSVEType(TypeFlags);
3895
3896 if (TypeFlags.isOverloadWhileOrMultiVecCvt())
3897 return {DefaultType, Ops[1]->getType()};
3898
3899 if (TypeFlags.isOverloadWhileRW())
3900 return {getSVEPredType(TypeFlags), Ops[0]->getType()};
3901
3902 if (TypeFlags.isOverloadDefaultAndOp0())
3903 return {DefaultType, Ops[0]->getType()};
3904
3905 if (TypeFlags.isOverloadFirstandLast())
3906 return {Ops[0]->getType(), Ops.back()->getType()};
3907
3908 if (TypeFlags.isReductionQV())
3909 return {ResultType, Ops[1]->getType()};
3910
3911 assert(TypeFlags.isOverloadDefault() && "Unexpected value for overloads");
3912 return {DefaultType};
3913}
3914
3915Value *CodeGenFunction::EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
3916 ArrayRef<Value *> Ops) {
3917 assert((TypeFlags.isTupleSet() || TypeFlags.isTupleGet()) &&
3918 "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()");
3919 unsigned Idx = cast<ConstantInt>(Val: Ops[1])->getZExtValue();
3920
3921 if (TypeFlags.isTupleSet())
3922 return Builder.CreateInsertValue(Agg: Ops[0], Val: Ops[2], Idxs: Idx);
3923 return Builder.CreateExtractValue(Agg: Ops[0], Idxs: Idx);
3924}
3925
3926Value *CodeGenFunction::EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
3927 llvm::Type *Ty,
3928 ArrayRef<Value *> Ops) {
3929 assert(TypeFlags.isTupleCreate() && "Expects TypleFlag isTupleCreate");
3930
3931 Value *Tuple = llvm::PoisonValue::get(T: Ty);
3932 for (unsigned Idx = 0; Idx < Ops.size(); Idx++)
3933 Tuple = Builder.CreateInsertValue(Agg: Tuple, Val: Ops[Idx], Idxs: Idx);
3934
3935 return Tuple;
3936}
3937
3938void CodeGenFunction::GetAArch64SVEProcessedOperands(
3939 unsigned BuiltinID, const CallExpr *E, SmallVectorImpl<Value *> &Ops,
3940 SVETypeFlags TypeFlags) {
3941 // Find out if any arguments are required to be integer constant expressions.
3942 unsigned ICEArguments = 0;
3943 ASTContext::GetBuiltinTypeError Error;
3944 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
3945 assert(Error == ASTContext::GE_None && "Should not codegen an error");
3946
3947 // Tuple set/get only requires one insert/extract vector, which is
3948 // created by EmitSVETupleSetOrGet.
3949 bool IsTupleGetOrSet = TypeFlags.isTupleSet() || TypeFlags.isTupleGet();
3950
3951 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
3952 bool IsICE = ICEArguments & (1 << i);
3953 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: i));
3954
3955 if (IsICE) {
3956 // If this is required to be a constant, constant fold it so that we know
3957 // that the generated intrinsic gets a ConstantInt.
3958 std::optional<llvm::APSInt> Result =
3959 E->getArg(Arg: i)->getIntegerConstantExpr(Ctx: getContext());
3960 assert(Result && "Expected argument to be a constant");
3961
3962 // Immediates for SVE llvm intrinsics are always 32bit. We can safely
3963 // truncate because the immediate has been range checked and no valid
3964 // immediate requires more than a handful of bits.
3965 *Result = Result->extOrTrunc(width: 32);
3966 Ops.push_back(Elt: llvm::ConstantInt::get(Context&: getLLVMContext(), V: *Result));
3967 continue;
3968 }
3969
3970 if (isa<StructType>(Val: Arg->getType()) && !IsTupleGetOrSet) {
3971 for (unsigned I = 0; I < Arg->getType()->getStructNumElements(); ++I)
3972 Ops.push_back(Elt: Builder.CreateExtractValue(Agg: Arg, Idxs: I));
3973
3974 continue;
3975 }
3976
3977 Ops.push_back(Elt: Arg);
3978 }
3979}
3980
3981Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
3982 const CallExpr *E) {
3983 llvm::Type *Ty = ConvertType(T: E->getType());
3984 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
3985 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
3986 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
3987 return EmitSVEReinterpret(Val, Ty);
3988 }
3989
3990 auto *Builtin =
3991 findARMVectorIntrinsicInMap(IntrinsicMap: ArrayRef(AArch64SVEIntrinsicMap), BuiltinID,
3992 MapProvenSorted&: AArch64SVEIntrinsicsProvenSorted);
3993
3994 llvm::SmallVector<Value *, 4> Ops;
3995 SVETypeFlags TypeFlags(Builtin->TypeModifier);
3996 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
3997
3998 if (TypeFlags.isLoad())
3999 return EmitSVEMaskedLoad(E, ReturnTy: Ty, Ops, IntrinsicID: Builtin->LLVMIntrinsic,
4000 IsZExtReturn: TypeFlags.isZExtReturn());
4001 if (TypeFlags.isStore())
4002 return EmitSVEMaskedStore(E, Ops, IntrinsicID: Builtin->LLVMIntrinsic);
4003 if (TypeFlags.isGatherLoad())
4004 return EmitSVEGatherLoad(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4005 if (TypeFlags.isScatterStore())
4006 return EmitSVEScatterStore(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4007 if (TypeFlags.isPrefetch())
4008 return EmitSVEPrefetchLoad(TypeFlags, Ops, BuiltinID: Builtin->LLVMIntrinsic);
4009 if (TypeFlags.isGatherPrefetch())
4010 return EmitSVEGatherPrefetch(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4011 if (TypeFlags.isStructLoad())
4012 return EmitSVEStructLoad(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4013 if (TypeFlags.isStructStore())
4014 return EmitSVEStructStore(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4015 if (TypeFlags.isTupleSet() || TypeFlags.isTupleGet())
4016 return EmitSVETupleSetOrGet(TypeFlags, Ops);
4017 if (TypeFlags.isTupleCreate())
4018 return EmitSVETupleCreate(TypeFlags, Ty, Ops);
4019 if (TypeFlags.isUndef())
4020 return UndefValue::get(T: Ty);
4021
4022 // Handle built-ins for which there is a corresponding LLVM Intrinsic.
4023 // -------------------------------------------------------------------
4024 if (Builtin->LLVMIntrinsic != 0) {
4025 // Emit set FPMR for intrinsics that require it
4026 if (TypeFlags.setsFPMR())
4027 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_set_fpmr),
4028 Args: Ops.pop_back_val());
4029 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZeroExp)
4030 InsertExplicitZeroOperand(Builder, Ty, Ops);
4031
4032 if (TypeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
4033 InsertExplicitUndefOperand(Builder, Ty, Ops);
4034
4035 // Some ACLE builtins leave out the argument to specify the predicate
4036 // pattern, which is expected to be expanded to an SV_ALL pattern.
4037 if (TypeFlags.isAppendSVALL())
4038 Ops.push_back(Elt: Builder.getInt32(/*SV_ALL*/ C: 31));
4039 if (TypeFlags.isInsertOp1SVALL())
4040 Ops.insert(I: &Ops[1], Elt: Builder.getInt32(/*SV_ALL*/ C: 31));
4041
4042 // Predicates must match the main datatype.
4043 for (Value *&Op : Ops)
4044 if (auto PredTy = dyn_cast<llvm::VectorType>(Val: Op->getType()))
4045 if (PredTy->getElementType()->isIntegerTy(BitWidth: 1))
4046 Op = EmitSVEPredicateCast(Pred: Op, VTy: getSVEType(TypeFlags));
4047
4048 // Splat scalar operand to vector (intrinsics with _n infix)
4049 if (TypeFlags.hasSplatOperand()) {
4050 unsigned OpNo = TypeFlags.getSplatOperand();
4051 Ops[OpNo] = EmitSVEDupX(Scalar: Ops[OpNo]);
4052 }
4053
4054 if (TypeFlags.isReverseCompare())
4055 std::swap(a&: Ops[1], b&: Ops[2]);
4056 else if (TypeFlags.isReverseUSDOT())
4057 std::swap(a&: Ops[1], b&: Ops[2]);
4058 else if (TypeFlags.isReverseMergeAnyBinOp() &&
4059 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4060 std::swap(a&: Ops[1], b&: Ops[2]);
4061 else if (TypeFlags.isReverseMergeAnyAccOp() &&
4062 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4063 std::swap(a&: Ops[1], b&: Ops[3]);
4064
4065 // Predicated intrinsics with _z suffix need a select w/ zeroinitializer.
4066 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZero) {
4067 llvm::Type *OpndTy = Ops[1]->getType();
4068 auto *SplatZero = Constant::getNullValue(Ty: OpndTy);
4069 Ops[1] = Builder.CreateSelect(C: Ops[0], True: Ops[1], False: SplatZero);
4070 }
4071
4072 Function *F = CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic,
4073 Tys: getSVEOverloadTypes(TypeFlags, ResultType: Ty, Ops));
4074 Value *Call = Builder.CreateCall(Callee: F, Args: Ops);
4075
4076 if (Call->getType() == Ty)
4077 return Call;
4078
4079 // Predicate results must be converted to svbool_t.
4080 if (auto PredTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty))
4081 return EmitSVEPredicateCast(Pred: Call, VTy: PredTy);
4082 if (auto PredTupleTy = dyn_cast<llvm::StructType>(Val: Ty))
4083 return EmitSVEPredicateTupleCast(PredTuple: Call, Ty: PredTupleTy);
4084
4085 llvm_unreachable("unsupported element count!");
4086 }
4087
4088 switch (BuiltinID) {
4089 default:
4090 return nullptr;
4091
4092 case SVE::BI__builtin_sve_svreinterpret_b: {
4093 Function *CastFromSVCountF =
4094 CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_convert_from_svcount);
4095 return Builder.CreateCall(Callee: CastFromSVCountF, Args: Ops[0]);
4096 }
4097 case SVE::BI__builtin_sve_svreinterpret_c: {
4098 Function *CastToSVCountF =
4099 CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_convert_to_svcount);
4100 return Builder.CreateCall(Callee: CastToSVCountF, Args: Ops[0]);
4101 }
4102
4103 case SVE::BI__builtin_sve_svpsel_lane_b8:
4104 case SVE::BI__builtin_sve_svpsel_lane_b16:
4105 case SVE::BI__builtin_sve_svpsel_lane_b32:
4106 case SVE::BI__builtin_sve_svpsel_lane_b64:
4107 case SVE::BI__builtin_sve_svpsel_lane_c8:
4108 case SVE::BI__builtin_sve_svpsel_lane_c16:
4109 case SVE::BI__builtin_sve_svpsel_lane_c32:
4110 case SVE::BI__builtin_sve_svpsel_lane_c64: {
4111 auto OverloadedTy = getSVEType(TypeFlags: SVETypeFlags(Builtin->TypeModifier));
4112 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_psel,
4113 Tys: {Ops[0]->getType(), OverloadedTy});
4114 llvm::Value *Ops1 = EmitSVEPredicateCast(Pred: Ops[1], VTy: OverloadedTy);
4115 return Builder.CreateCall(Callee: F, Args: {Ops[0], Ops1, Ops[2]});
4116 }
4117 case SVE::BI__builtin_sve_svmov_b_z: {
4118 // svmov_b_z(pg, op) <=> svand_b_z(pg, op, op)
4119 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4120 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4121 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_and_z, Tys: OverloadedTy);
4122 return Builder.CreateCall(Callee: F, Args: {Ops[0], Ops[1], Ops[1]});
4123 }
4124
4125 case SVE::BI__builtin_sve_svnot_b_z: {
4126 // svnot_b_z(pg, op) <=> sveor_b_z(pg, op, pg)
4127 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4128 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4129 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_eor_z, Tys: OverloadedTy);
4130 return Builder.CreateCall(Callee: F, Args: {Ops[0], Ops[1], Ops[0]});
4131 }
4132
4133 case SVE::BI__builtin_sve_svmovlb_u16:
4134 case SVE::BI__builtin_sve_svmovlb_u32:
4135 case SVE::BI__builtin_sve_svmovlb_u64:
4136 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_ushllb);
4137
4138 case SVE::BI__builtin_sve_svmovlb_s16:
4139 case SVE::BI__builtin_sve_svmovlb_s32:
4140 case SVE::BI__builtin_sve_svmovlb_s64:
4141 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_sshllb);
4142
4143 case SVE::BI__builtin_sve_svmovlt_u16:
4144 case SVE::BI__builtin_sve_svmovlt_u32:
4145 case SVE::BI__builtin_sve_svmovlt_u64:
4146 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_ushllt);
4147
4148 case SVE::BI__builtin_sve_svmovlt_s16:
4149 case SVE::BI__builtin_sve_svmovlt_s32:
4150 case SVE::BI__builtin_sve_svmovlt_s64:
4151 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_sshllt);
4152
4153 case SVE::BI__builtin_sve_svpmullt_u16:
4154 case SVE::BI__builtin_sve_svpmullt_u64:
4155 case SVE::BI__builtin_sve_svpmullt_n_u16:
4156 case SVE::BI__builtin_sve_svpmullt_n_u64:
4157 return EmitSVEPMull(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_pmullt_pair);
4158
4159 case SVE::BI__builtin_sve_svpmullb_u16:
4160 case SVE::BI__builtin_sve_svpmullb_u64:
4161 case SVE::BI__builtin_sve_svpmullb_n_u16:
4162 case SVE::BI__builtin_sve_svpmullb_n_u64:
4163 return EmitSVEPMull(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_pmullb_pair);
4164
4165 case SVE::BI__builtin_sve_svdup_n_b8:
4166 case SVE::BI__builtin_sve_svdup_n_b16:
4167 case SVE::BI__builtin_sve_svdup_n_b32:
4168 case SVE::BI__builtin_sve_svdup_n_b64: {
4169 llvm::ScalableVectorType *OverloadedTy = getSVEType(TypeFlags);
4170 Value *Dup = EmitSVEDupX(Scalar: Ops[0], Ty: OverloadedTy);
4171 return EmitSVEPredicateCast(Pred: Dup, VTy: cast<llvm::ScalableVectorType>(Val: Ty));
4172 }
4173
4174 case SVE::BI__builtin_sve_svdupq_n_b8:
4175 case SVE::BI__builtin_sve_svdupq_n_b16:
4176 case SVE::BI__builtin_sve_svdupq_n_b32:
4177 case SVE::BI__builtin_sve_svdupq_n_b64:
4178 case SVE::BI__builtin_sve_svdupq_n_u8:
4179 case SVE::BI__builtin_sve_svdupq_n_s8:
4180 case SVE::BI__builtin_sve_svdupq_n_u64:
4181 case SVE::BI__builtin_sve_svdupq_n_f64:
4182 case SVE::BI__builtin_sve_svdupq_n_s64:
4183 case SVE::BI__builtin_sve_svdupq_n_u16:
4184 case SVE::BI__builtin_sve_svdupq_n_f16:
4185 case SVE::BI__builtin_sve_svdupq_n_bf16:
4186 case SVE::BI__builtin_sve_svdupq_n_s16:
4187 case SVE::BI__builtin_sve_svdupq_n_u32:
4188 case SVE::BI__builtin_sve_svdupq_n_f32:
4189 case SVE::BI__builtin_sve_svdupq_n_s32: {
4190 // These builtins are implemented by storing each element to an array and using
4191 // ld1rq to materialize a vector.
4192 unsigned NumOpnds = Ops.size();
4193
4194 bool IsBoolTy =
4195 cast<llvm::VectorType>(Val: Ty)->getElementType()->isIntegerTy(BitWidth: 1);
4196
4197 // For svdupq_n_b* the element type of is an integer of type 128/numelts,
4198 // so that the compare can use the width that is natural for the expected
4199 // number of predicate lanes.
4200 llvm::Type *EltTy = Ops[0]->getType();
4201 if (IsBoolTy)
4202 EltTy = IntegerType::get(C&: getLLVMContext(), NumBits: SVEBitsPerBlock / NumOpnds);
4203
4204 SmallVector<llvm::Value *, 16> VecOps;
4205 for (unsigned I = 0; I < NumOpnds; ++I)
4206 VecOps.push_back(Elt: Builder.CreateZExt(V: Ops[I], DestTy: EltTy));
4207 Value *Vec = BuildVector(Ops: VecOps);
4208
4209 llvm::Type *OverloadedTy = getSVEVectorForElementType(EltTy);
4210 Value *InsertSubVec = Builder.CreateInsertVector(
4211 DstType: OverloadedTy, SrcVec: PoisonValue::get(T: OverloadedTy), SubVec: Vec, Idx: uint64_t(0));
4212
4213 Function *F =
4214 CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_dupq_lane, Tys: OverloadedTy);
4215 Value *DupQLane =
4216 Builder.CreateCall(Callee: F, Args: {InsertSubVec, Builder.getInt64(C: 0)});
4217
4218 if (!IsBoolTy)
4219 return DupQLane;
4220
4221 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4222 Constant *Pred = ConstantInt::getTrue(Ty: getSVEPredType(TypeFlags));
4223
4224 // For svdupq_n_b* we need to add an additional 'cmpne' with '0'.
4225 F = CGM.getIntrinsic(IID: NumOpnds == 2 ? Intrinsic::aarch64_sve_cmpne
4226 : Intrinsic::aarch64_sve_cmpne_wide,
4227 Tys: OverloadedTy);
4228 Value *Call = Builder.CreateCall(
4229 Callee: F, Args: {Pred, DupQLane, EmitSVEDupX(Scalar: Builder.getInt64(C: 0))});
4230 return EmitSVEPredicateCast(Pred: Call, VTy: cast<llvm::ScalableVectorType>(Val: Ty));
4231 }
4232
4233 case SVE::BI__builtin_sve_svpfalse_b:
4234 return ConstantInt::getFalse(Ty);
4235
4236 case SVE::BI__builtin_sve_svpfalse_c:
4237 return Constant::getNullValue(Ty);
4238
4239 case SVE::BI__builtin_sve_svlen_bf16:
4240 case SVE::BI__builtin_sve_svlen_f16:
4241 case SVE::BI__builtin_sve_svlen_f32:
4242 case SVE::BI__builtin_sve_svlen_f64:
4243 case SVE::BI__builtin_sve_svlen_s8:
4244 case SVE::BI__builtin_sve_svlen_s16:
4245 case SVE::BI__builtin_sve_svlen_s32:
4246 case SVE::BI__builtin_sve_svlen_s64:
4247 case SVE::BI__builtin_sve_svlen_u8:
4248 case SVE::BI__builtin_sve_svlen_u16:
4249 case SVE::BI__builtin_sve_svlen_u32:
4250 case SVE::BI__builtin_sve_svlen_u64: {
4251 SVETypeFlags TF(Builtin->TypeModifier);
4252 return Builder.CreateElementCount(Ty, EC: getSVEType(TypeFlags: TF)->getElementCount());
4253 }
4254
4255 case SVE::BI__builtin_sve_svtbl2_u8:
4256 case SVE::BI__builtin_sve_svtbl2_s8:
4257 case SVE::BI__builtin_sve_svtbl2_u16:
4258 case SVE::BI__builtin_sve_svtbl2_s16:
4259 case SVE::BI__builtin_sve_svtbl2_u32:
4260 case SVE::BI__builtin_sve_svtbl2_s32:
4261 case SVE::BI__builtin_sve_svtbl2_u64:
4262 case SVE::BI__builtin_sve_svtbl2_s64:
4263 case SVE::BI__builtin_sve_svtbl2_f16:
4264 case SVE::BI__builtin_sve_svtbl2_bf16:
4265 case SVE::BI__builtin_sve_svtbl2_f32:
4266 case SVE::BI__builtin_sve_svtbl2_f64: {
4267 SVETypeFlags TF(Builtin->TypeModifier);
4268 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_tbl2, Tys: getSVEType(TypeFlags: TF));
4269 return Builder.CreateCall(Callee: F, Args: Ops);
4270 }
4271
4272 case SVE::BI__builtin_sve_svset_neonq_s8:
4273 case SVE::BI__builtin_sve_svset_neonq_s16:
4274 case SVE::BI__builtin_sve_svset_neonq_s32:
4275 case SVE::BI__builtin_sve_svset_neonq_s64:
4276 case SVE::BI__builtin_sve_svset_neonq_u8:
4277 case SVE::BI__builtin_sve_svset_neonq_u16:
4278 case SVE::BI__builtin_sve_svset_neonq_u32:
4279 case SVE::BI__builtin_sve_svset_neonq_u64:
4280 case SVE::BI__builtin_sve_svset_neonq_f16:
4281 case SVE::BI__builtin_sve_svset_neonq_f32:
4282 case SVE::BI__builtin_sve_svset_neonq_f64:
4283 case SVE::BI__builtin_sve_svset_neonq_bf16:
4284 case SVE::BI__builtin_sve_svset_neonq_mf8: {
4285 return Builder.CreateInsertVector(DstType: Ty, SrcVec: Ops[0], SubVec: Ops[1], Idx: uint64_t(0));
4286 }
4287
4288 case SVE::BI__builtin_sve_svget_neonq_s8:
4289 case SVE::BI__builtin_sve_svget_neonq_s16:
4290 case SVE::BI__builtin_sve_svget_neonq_s32:
4291 case SVE::BI__builtin_sve_svget_neonq_s64:
4292 case SVE::BI__builtin_sve_svget_neonq_u8:
4293 case SVE::BI__builtin_sve_svget_neonq_u16:
4294 case SVE::BI__builtin_sve_svget_neonq_u32:
4295 case SVE::BI__builtin_sve_svget_neonq_u64:
4296 case SVE::BI__builtin_sve_svget_neonq_f16:
4297 case SVE::BI__builtin_sve_svget_neonq_f32:
4298 case SVE::BI__builtin_sve_svget_neonq_f64:
4299 case SVE::BI__builtin_sve_svget_neonq_bf16:
4300 case SVE::BI__builtin_sve_svget_neonq_mf8: {
4301 return Builder.CreateExtractVector(DstType: Ty, SrcVec: Ops[0], Idx: uint64_t(0));
4302 }
4303
4304 case SVE::BI__builtin_sve_svdup_neonq_s8:
4305 case SVE::BI__builtin_sve_svdup_neonq_s16:
4306 case SVE::BI__builtin_sve_svdup_neonq_s32:
4307 case SVE::BI__builtin_sve_svdup_neonq_s64:
4308 case SVE::BI__builtin_sve_svdup_neonq_u8:
4309 case SVE::BI__builtin_sve_svdup_neonq_u16:
4310 case SVE::BI__builtin_sve_svdup_neonq_u32:
4311 case SVE::BI__builtin_sve_svdup_neonq_u64:
4312 case SVE::BI__builtin_sve_svdup_neonq_f16:
4313 case SVE::BI__builtin_sve_svdup_neonq_f32:
4314 case SVE::BI__builtin_sve_svdup_neonq_f64:
4315 case SVE::BI__builtin_sve_svdup_neonq_bf16:
4316 case SVE::BI__builtin_sve_svdup_neonq_mf8: {
4317 Value *Insert = Builder.CreateInsertVector(DstType: Ty, SrcVec: PoisonValue::get(T: Ty), SubVec: Ops[0],
4318 Idx: uint64_t(0));
4319 return Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_dupq_lane, OverloadTypes: {Ty},
4320 Args: {Insert, Builder.getInt64(C: 0)});
4321 }
4322 }
4323
4324 /// Should not happen
4325 return nullptr;
4326}
4327
4328static void swapCommutativeSMEOperands(unsigned BuiltinID,
4329 SmallVectorImpl<Value *> &Ops) {
4330 unsigned MultiVec;
4331 switch (BuiltinID) {
4332 default:
4333 return;
4334 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
4335 MultiVec = 1;
4336 break;
4337 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
4338 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
4339 MultiVec = 2;
4340 break;
4341 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
4342 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
4343 MultiVec = 4;
4344 break;
4345 }
4346
4347 if (MultiVec > 0)
4348 for (unsigned I = 0; I < MultiVec; ++I)
4349 std::swap(a&: Ops[I + 1], b&: Ops[I + 1 + MultiVec]);
4350}
4351
4352Value *CodeGenFunction::EmitAArch64SMEBuiltinExpr(unsigned BuiltinID,
4353 const CallExpr *E) {
4354 auto *Builtin =
4355 findARMVectorIntrinsicInMap(IntrinsicMap: ArrayRef(AArch64SMEIntrinsicMap), BuiltinID,
4356 MapProvenSorted&: AArch64SMEIntrinsicsProvenSorted);
4357
4358 llvm::SmallVector<Value *, 4> Ops;
4359 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4360 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
4361
4362 if (TypeFlags.isLoad() || TypeFlags.isStore())
4363 return EmitSMELd1St1(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4364 if (TypeFlags.isReadZA() || TypeFlags.isWriteZA())
4365 return EmitSMEReadWrite(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4366 if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
4367 BuiltinID == SME::BI__builtin_sme_svzero_za)
4368 return EmitSMEZero(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4369 if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
4370 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
4371 BuiltinID == SME::BI__builtin_sme_svldr_za ||
4372 BuiltinID == SME::BI__builtin_sme_svstr_za)
4373 return EmitSMELdrStr(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4374
4375 // Emit set FPMR for intrinsics that require it
4376 if (TypeFlags.setsFPMR())
4377 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_set_fpmr),
4378 Args: Ops.pop_back_val());
4379 // Handle builtins which require their multi-vector operands to be swapped
4380 swapCommutativeSMEOperands(BuiltinID, Ops);
4381
4382 auto isCntsBuiltin = [&]() {
4383 switch (BuiltinID) {
4384 default:
4385 return 0;
4386 case SME::BI__builtin_sme_svcntsb:
4387 return 8;
4388 case SME::BI__builtin_sme_svcntsh:
4389 return 4;
4390 case SME::BI__builtin_sme_svcntsw:
4391 return 2;
4392 }
4393 };
4394
4395 if (auto Mul = isCntsBuiltin()) {
4396 llvm::Value *Cntd =
4397 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_sme_cntsd));
4398 return Builder.CreateMul(LHS: Cntd, RHS: llvm::ConstantInt::get(Ty: Int64Ty, V: Mul),
4399 Name: "mulsvl", /* HasNUW */ true, /* HasNSW */ true);
4400 }
4401
4402 // Should not happen!
4403 if (Builtin->LLVMIntrinsic == 0)
4404 return nullptr;
4405
4406 // Predicates must match the main datatype.
4407 for (Value *&Op : Ops)
4408 if (auto PredTy = dyn_cast<llvm::VectorType>(Val: Op->getType()))
4409 if (PredTy->getElementType()->isIntegerTy(BitWidth: 1))
4410 Op = EmitSVEPredicateCast(Pred: Op, VTy: getSVEType(TypeFlags));
4411
4412 if (BuiltinID == SME::BI__builtin_sme_svldr_zt ||
4413 BuiltinID == SME::BI__builtin_sme_svstr_zt) {
4414 Function *F = CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic, Tys: Ops[1]->getType());
4415 return Builder.CreateCall(Callee: F, Args: Ops);
4416 }
4417
4418 Function *F =
4419 TypeFlags.isOverloadNone()
4420 ? CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic)
4421 : CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic, Tys: {getSVEType(TypeFlags)});
4422
4423 return Builder.CreateCall(Callee: F, Args: Ops);
4424}
4425
4426/// Helper for the read/write/add/inc X18 builtins: read the X18 register and
4427/// return it as an i8 pointer.
4428Value *readX18AsPtr(CodeGenFunction &CGF) {
4429 LLVMContext &Context = CGF.CGM.getLLVMContext();
4430 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Str: "x18")};
4431 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
4432 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
4433 llvm::Function *F =
4434 CGF.CGM.getIntrinsic(IID: Intrinsic::read_register, Tys: {CGF.Int64Ty});
4435 llvm::Value *X18 = CGF.Builder.CreateCall(Callee: F, Args: Metadata);
4436 return CGF.Builder.CreateIntToPtr(V: X18, DestTy: CGF.Int8PtrTy);
4437}
4438
4439Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
4440 const CallExpr *E,
4441 llvm::Triple::ArchType Arch) {
4442 if (BuiltinID >= clang::AArch64::FirstSVEBuiltin &&
4443 BuiltinID <= clang::AArch64::LastSVEBuiltin)
4444 return EmitAArch64SVEBuiltinExpr(BuiltinID, E);
4445
4446 if (BuiltinID >= clang::AArch64::FirstSMEBuiltin &&
4447 BuiltinID <= clang::AArch64::LastSMEBuiltin)
4448 return EmitAArch64SMEBuiltinExpr(BuiltinID, E);
4449
4450 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
4451 return EmitAArch64CpuSupports(E);
4452
4453 unsigned HintID = static_cast<unsigned>(-1);
4454 switch (BuiltinID) {
4455 default: break;
4456 case clang::AArch64::BI__builtin_arm_nop:
4457 HintID = 0;
4458 break;
4459 case clang::AArch64::BI__builtin_arm_yield:
4460 case clang::AArch64::BI__yield:
4461 HintID = 1;
4462 break;
4463 case clang::AArch64::BI__builtin_arm_wfe:
4464 case clang::AArch64::BI__wfe:
4465 HintID = 2;
4466 break;
4467 case clang::AArch64::BI__builtin_arm_wfi:
4468 case clang::AArch64::BI__wfi:
4469 HintID = 3;
4470 break;
4471 case clang::AArch64::BI__builtin_arm_sev:
4472 case clang::AArch64::BI__sev:
4473 HintID = 4;
4474 break;
4475 case clang::AArch64::BI__builtin_arm_sevl:
4476 case clang::AArch64::BI__sevl:
4477 HintID = 5;
4478 break;
4479 }
4480
4481 if (HintID != static_cast<unsigned>(-1)) {
4482 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_hint);
4483 return Builder.CreateCall(Callee: F, Args: llvm::ConstantInt::get(Ty: Int32Ty, V: HintID));
4484 }
4485
4486 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
4487 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_break);
4488 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4489 return Builder.CreateCall(Callee: F, Args: Builder.CreateZExt(V: Arg, DestTy: CGM.Int32Ty));
4490 }
4491
4492 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
4493 // Create call to __arm_sme_state and store the results to the two pointers.
4494 CallInst *CI = EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(
4495 Ty: llvm::FunctionType::get(Result: StructType::get(elt1: CGM.Int64Ty, elts: CGM.Int64Ty), Params: {},
4496 isVarArg: false),
4497 Name: "__arm_sme_state"));
4498 auto Attrs = AttributeList().addFnAttribute(C&: getLLVMContext(),
4499 Kind: "aarch64_pstate_sm_compatible");
4500 CI->setAttributes(Attrs);
4501 CI->setCallingConv(
4502 llvm::CallingConv::
4503 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
4504 Builder.CreateStore(Val: Builder.CreateExtractValue(Agg: CI, Idxs: 0),
4505 Addr: EmitPointerWithAlignment(Addr: E->getArg(Arg: 0)));
4506 return Builder.CreateStore(Val: Builder.CreateExtractValue(Agg: CI, Idxs: 1),
4507 Addr: EmitPointerWithAlignment(Addr: E->getArg(Arg: 1)));
4508 }
4509
4510 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
4511 assert((getContext().getTypeSize(E->getType()) == 32) &&
4512 "rbit of unusual size!");
4513 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4514 return Builder.CreateCall(
4515 Callee: CGM.getIntrinsic(IID: Intrinsic::bitreverse, Tys: Arg->getType()), Args: Arg, Name: "rbit");
4516 }
4517 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
4518 assert((getContext().getTypeSize(E->getType()) == 64) &&
4519 "rbit of unusual size!");
4520 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4521 return Builder.CreateCall(
4522 Callee: CGM.getIntrinsic(IID: Intrinsic::bitreverse, Tys: Arg->getType()), Args: Arg, Name: "rbit");
4523 }
4524
4525 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
4526 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
4527 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4528 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: Arg->getType());
4529 Value *Res = Builder.CreateCall(Callee: F, Args: {Arg, Builder.getInt1(V: false)});
4530 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
4531 Res = Builder.CreateTrunc(V: Res, DestTy: Builder.getInt32Ty());
4532 return Res;
4533 }
4534
4535 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
4536 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4537 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_cls), Args: Arg,
4538 Name: "cls");
4539 }
4540 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
4541 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4542 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_cls64), Args: Arg,
4543 Name: "cls");
4544 }
4545
4546 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
4547 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
4548 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4549 llvm::Type *Ty = Arg->getType();
4550 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint32z, Tys: Ty),
4551 Args: Arg, Name: "frint32z");
4552 }
4553
4554 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
4555 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
4556 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4557 llvm::Type *Ty = Arg->getType();
4558 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint64z, Tys: Ty),
4559 Args: Arg, Name: "frint64z");
4560 }
4561
4562 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
4563 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
4564 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4565 llvm::Type *Ty = Arg->getType();
4566 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint32x, Tys: Ty),
4567 Args: Arg, Name: "frint32x");
4568 }
4569
4570 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
4571 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
4572 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4573 llvm::Type *Ty = Arg->getType();
4574 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint64x, Tys: Ty),
4575 Args: Arg, Name: "frint64x");
4576 }
4577
4578 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
4579 assert((getContext().getTypeSize(E->getType()) == 32) &&
4580 "__jcvt of unusual size!");
4581 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4582 return Builder.CreateCall(
4583 Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_fjcvtzs), Args: Arg);
4584 }
4585
4586 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
4587 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
4588 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
4589 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
4590 llvm::Value *MemAddr = EmitScalarExpr(E: E->getArg(Arg: 0));
4591 llvm::Value *ValPtr = EmitScalarExpr(E: E->getArg(Arg: 1));
4592
4593 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
4594 // Load from the address via an LLVM intrinsic, receiving a
4595 // tuple of 8 i64 words, and store each one to ValPtr.
4596 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_ld64b);
4597 llvm::Value *Val = Builder.CreateCall(Callee: F, Args: MemAddr);
4598 llvm::Value *ToRet;
4599 for (size_t i = 0; i < 8; i++) {
4600 llvm::Value *ValOffsetPtr =
4601 Builder.CreateGEP(Ty: Int64Ty, Ptr: ValPtr, IdxList: Builder.getInt32(C: i));
4602 Address Addr =
4603 Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(Quantity: 8));
4604 ToRet = Builder.CreateStore(Val: Builder.CreateExtractValue(Agg: Val, Idxs: i), Addr);
4605 }
4606 return ToRet;
4607 }
4608
4609 // Load 8 i64 words from ValPtr, and store them to the address
4610 // via an LLVM intrinsic.
4611 SmallVector<llvm::Value *, 9> Args;
4612 Args.push_back(Elt: MemAddr);
4613 for (size_t i = 0; i < 8; i++) {
4614 llvm::Value *ValOffsetPtr =
4615 Builder.CreateGEP(Ty: Int64Ty, Ptr: ValPtr, IdxList: Builder.getInt32(C: i));
4616 Address Addr = Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(Quantity: 8));
4617 Args.push_back(Elt: Builder.CreateLoad(Addr));
4618 }
4619
4620 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
4621 ? Intrinsic::aarch64_st64b
4622 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
4623 ? Intrinsic::aarch64_st64bv
4624 : Intrinsic::aarch64_st64bv0);
4625 Function *F = CGM.getIntrinsic(IID: Intr);
4626 return Builder.CreateCall(Callee: F, Args);
4627 }
4628
4629 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
4630 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
4631
4632 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
4633 ? Intrinsic::aarch64_rndr
4634 : Intrinsic::aarch64_rndrrs);
4635 Function *F = CGM.getIntrinsic(IID: Intr);
4636 llvm::Value *Val = Builder.CreateCall(Callee: F);
4637 Value *RandomValue = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
4638 Value *Status = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
4639
4640 Address MemAddress = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4641 Builder.CreateStore(Val: RandomValue, Addr: MemAddress);
4642 Status = Builder.CreateZExt(V: Status, DestTy: Int32Ty);
4643 return Status;
4644 }
4645
4646 if (BuiltinID == clang::AArch64::BI__clear_cache) {
4647 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
4648 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
4649 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache, Tys: {CGM.DefaultPtrTy});
4650 return Builder.CreateCall(Callee: F, Args: {Begin, End});
4651 }
4652
4653 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4654 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
4655 getContext().getTypeSize(T: E->getType()) == 128) {
4656 Function *F =
4657 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4658 ? Intrinsic::aarch64_ldaxp
4659 : Intrinsic::aarch64_ldxp);
4660
4661 Value *LdPtr = EmitScalarExpr(E: E->getArg(Arg: 0));
4662 Value *Val = Builder.CreateCall(Callee: F, Args: LdPtr, Name: "ldxp");
4663
4664 Value *Val0 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
4665 Value *Val1 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
4666 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 128);
4667 Val0 = Builder.CreateZExt(V: Val0, DestTy: Int128Ty);
4668 Val1 = Builder.CreateZExt(V: Val1, DestTy: Int128Ty);
4669
4670 Value *ShiftCst = llvm::ConstantInt::get(Ty: Int128Ty, V: 64);
4671 Val = Builder.CreateShl(LHS: Val0, RHS: ShiftCst, Name: "shl", HasNUW: true /* nuw */);
4672 Val = Builder.CreateOr(LHS: Val, RHS: Val1);
4673 return Builder.CreateBitCast(V: Val, DestTy: ConvertType(T: E->getType()));
4674 } else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4675 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
4676 Value *LoadAddr = EmitScalarExpr(E: E->getArg(Arg: 0));
4677
4678 QualType Ty = E->getType();
4679 llvm::Type *RealResTy = ConvertType(T: Ty);
4680 llvm::Type *IntTy =
4681 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
4682
4683 Function *F =
4684 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4685 ? Intrinsic::aarch64_ldaxr
4686 : Intrinsic::aarch64_ldxr,
4687 Tys: DefaultPtrTy);
4688 CallInst *Val = Builder.CreateCall(Callee: F, Args: LoadAddr, Name: "ldxr");
4689 Val->addParamAttr(
4690 ArgNo: 0, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: IntTy));
4691
4692 if (RealResTy->isPointerTy())
4693 return Builder.CreateIntToPtr(V: Val, DestTy: RealResTy);
4694
4695 llvm::Type *IntResTy = llvm::IntegerType::get(
4696 C&: getLLVMContext(), NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: RealResTy));
4697 return Builder.CreateBitCast(V: Builder.CreateTruncOrBitCast(V: Val, DestTy: IntResTy),
4698 DestTy: RealResTy);
4699 }
4700
4701 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4702 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
4703 getContext().getTypeSize(T: E->getArg(Arg: 0)->getType()) == 128) {
4704 Function *F =
4705 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4706 ? Intrinsic::aarch64_stlxp
4707 : Intrinsic::aarch64_stxp);
4708 llvm::Type *STy = llvm::StructType::get(elt1: Int64Ty, elts: Int64Ty);
4709
4710 Address Tmp = CreateMemTempWithoutCast(T: E->getArg(Arg: 0)->getType());
4711 EmitAnyExprToMem(E: E->getArg(Arg: 0), Location: Tmp, Quals: Qualifiers(), /*init*/ IsInitializer: true);
4712
4713 Tmp = Tmp.withElementType(ElemTy: STy);
4714 llvm::Value *Val = Builder.CreateLoad(Addr: Tmp);
4715
4716 Value *Arg0 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
4717 Value *Arg1 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
4718 Value *StPtr = EmitScalarExpr(E: E->getArg(Arg: 1));
4719 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1, StPtr}, Name: "stxp");
4720 }
4721
4722 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4723 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
4724 Value *StoreVal = EmitScalarExpr(E: E->getArg(Arg: 0));
4725 Value *StoreAddr = EmitScalarExpr(E: E->getArg(Arg: 1));
4726
4727 QualType Ty = E->getArg(Arg: 0)->getType();
4728 llvm::Type *StoreTy =
4729 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
4730
4731 if (StoreVal->getType()->isPointerTy())
4732 StoreVal = Builder.CreatePtrToInt(V: StoreVal, DestTy: Int64Ty);
4733 else {
4734 llvm::Type *IntTy = llvm::IntegerType::get(
4735 C&: getLLVMContext(),
4736 NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: StoreVal->getType()));
4737 StoreVal = Builder.CreateBitCast(V: StoreVal, DestTy: IntTy);
4738 StoreVal = Builder.CreateZExtOrBitCast(V: StoreVal, DestTy: Int64Ty);
4739 }
4740
4741 Function *F =
4742 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4743 ? Intrinsic::aarch64_stlxr
4744 : Intrinsic::aarch64_stxr,
4745 Tys: StoreAddr->getType());
4746 CallInst *CI = Builder.CreateCall(Callee: F, Args: {StoreVal, StoreAddr}, Name: "stxr");
4747 CI->addParamAttr(
4748 ArgNo: 1, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: StoreTy));
4749 return CI;
4750 }
4751
4752 if (BuiltinID == clang::AArch64::BI__getReg ||
4753 BuiltinID == clang::AArch64::BI__setReg) {
4754 Expr::EvalResult Result;
4755 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4756 llvm_unreachable("Sema will ensure that the parameter is constant");
4757
4758 llvm::APSInt Value = Result.Val.getInt();
4759 LLVMContext &Context = CGM.getLLVMContext();
4760 std::string Reg = Value == 31 ? "sp" : "x" + toString(I: Value, Radix: 10);
4761
4762 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Str: Reg)};
4763 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
4764 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
4765
4766 CallInst *CI;
4767 if (BuiltinID == clang::AArch64::BI__getReg) {
4768 llvm::Function *F =
4769 CGM.getIntrinsic(IID: Intrinsic::read_volatile_register, Tys: {Int64Ty});
4770 CI = Builder.CreateCall(Callee: F, Args: Metadata);
4771 } else {
4772 llvm::Function *F =
4773 CGM.getIntrinsic(IID: Intrinsic::write_volatile_register, Tys: {Int64Ty});
4774 CI = Builder.CreateCall(Callee: F, Args: {Metadata, EmitScalarExpr(E: E->getArg(Arg: 1))});
4775 }
4776 return CI;
4777 }
4778
4779 if (BuiltinID == clang::AArch64::BI__getRegFp ||
4780 BuiltinID == clang::AArch64::BI__setRegFp) {
4781 Expr::EvalResult Result;
4782 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4783 llvm_unreachable("Sema will ensure that the parameter is constant");
4784
4785 llvm::APSInt Value = Result.Val.getInt();
4786 LLVMContext &Context = CGM.getLLVMContext();
4787 std::string Reg = "d" + toString(I: Value, Radix: 10);
4788
4789 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Str: Reg)};
4790 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
4791 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
4792
4793 llvm::Value *Ret;
4794 if (BuiltinID == clang::AArch64::BI__getRegFp) {
4795 llvm::Function *F =
4796 CGM.getIntrinsic(IID: Intrinsic::read_volatile_register, Tys: {Int64Ty});
4797 llvm::Value *Bits = Builder.CreateCall(Callee: F, Args: Metadata);
4798 Ret = Builder.CreateBitCast(V: Bits, DestTy: llvm::Type::getDoubleTy(C&: Context));
4799 } else {
4800 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
4801 llvm::Value *Bits = Builder.CreateBitCast(V: Val, DestTy: Int64Ty);
4802 llvm::Function *F =
4803 CGM.getIntrinsic(IID: Intrinsic::write_volatile_register, Tys: {Int64Ty});
4804 Ret = Builder.CreateCall(Callee: F, Args: {Metadata, Bits});
4805 }
4806 return Ret;
4807 }
4808
4809 if (BuiltinID == clang::AArch64::BI__break) {
4810 Expr::EvalResult Result;
4811 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4812 llvm_unreachable("Sema will ensure that the parameter is constant");
4813
4814 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_break);
4815 return Builder.CreateCall(Callee: F, Args: {EmitScalarExpr(E: E->getArg(Arg: 0))});
4816 }
4817
4818 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
4819 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_clrex);
4820 return Builder.CreateCall(Callee: F);
4821 }
4822
4823 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
4824 return Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
4825 SSID: llvm::SyncScope::SingleThread);
4826
4827 // CRC32
4828 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
4829 switch (BuiltinID) {
4830 case clang::AArch64::BI__builtin_arm_crc32b:
4831 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
4832 case clang::AArch64::BI__builtin_arm_crc32cb:
4833 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
4834 case clang::AArch64::BI__builtin_arm_crc32h:
4835 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
4836 case clang::AArch64::BI__builtin_arm_crc32ch:
4837 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
4838 case clang::AArch64::BI__builtin_arm_crc32w:
4839 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
4840 case clang::AArch64::BI__builtin_arm_crc32cw:
4841 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
4842 case clang::AArch64::BI__builtin_arm_crc32d:
4843 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
4844 case clang::AArch64::BI__builtin_arm_crc32cd:
4845 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
4846 }
4847
4848 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4849 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4850 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4851 Function *F = CGM.getIntrinsic(IID: CRCIntrinsicID);
4852
4853 llvm::Type *DataTy = F->getFunctionType()->getParamType(i: 1);
4854 Arg1 = Builder.CreateZExtOrBitCast(V: Arg1, DestTy: DataTy);
4855
4856 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1});
4857 }
4858
4859 // Memory Operations (MOPS)
4860 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
4861 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
4862 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
4863 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
4864 Val = Builder.CreateTrunc(V: Val, DestTy: Int8Ty);
4865 Size = Builder.CreateIntCast(V: Size, DestTy: Int64Ty, isSigned: false);
4866 return Builder.CreateCall(
4867 Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_mops_memset_tag), Args: {Dst, Val, Size});
4868 }
4869
4870 if (BuiltinID == AArch64::BI__builtin_arm_range_prefetch ||
4871 BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
4872 return EmitRangePrefetchBuiltin(CGF&: *this, BuiltinID, E);
4873
4874 if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
4875 return EmitAtomicStoreWithHintBuiltin(CGF&: *this, BuiltinID, E);
4876
4877 // Memory Tagging Extensions (MTE) Intrinsics
4878 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
4879 switch (BuiltinID) {
4880 case clang::AArch64::BI__builtin_arm_irg:
4881 MTEIntrinsicID = Intrinsic::aarch64_irg; break;
4882 case clang::AArch64::BI__builtin_arm_addg:
4883 MTEIntrinsicID = Intrinsic::aarch64_addg; break;
4884 case clang::AArch64::BI__builtin_arm_gmi:
4885 MTEIntrinsicID = Intrinsic::aarch64_gmi; break;
4886 case clang::AArch64::BI__builtin_arm_ldg:
4887 MTEIntrinsicID = Intrinsic::aarch64_ldg; break;
4888 case clang::AArch64::BI__builtin_arm_stg:
4889 MTEIntrinsicID = Intrinsic::aarch64_stg; break;
4890 case clang::AArch64::BI__builtin_arm_subp:
4891 MTEIntrinsicID = Intrinsic::aarch64_subp; break;
4892 }
4893
4894 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_lscale) {
4895 Value *FPM = EmitScalarExpr(E: E->getArg(Arg: 0));
4896 Value *Scale = EmitScalarExpr(E: E->getArg(Arg: 1));
4897 Scale = Builder.CreateAnd(LHS: Scale, RHS: Builder.getInt64(C: 0x7f));
4898
4899 Value *MaskedFPM = Builder.CreateAnd(LHS: FPM, RHS: Builder.getInt64(C: ~0x7f0000ULL));
4900 Value *ShiftedScale = Builder.CreateShl(LHS: Scale, RHS: Builder.getInt64(C: 16));
4901
4902 return Builder.CreateOr(LHS: MaskedFPM, RHS: ShiftedScale);
4903 }
4904 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_nscale) {
4905 Value *FPM = EmitScalarExpr(E: E->getArg(Arg: 0));
4906 Value *Scale = EmitScalarExpr(E: E->getArg(Arg: 1));
4907 Scale = Builder.CreateAnd(LHS: Scale, RHS: Builder.getInt64(C: 0xff));
4908
4909 Value *MaskedFPM = Builder.CreateAnd(LHS: FPM, RHS: Builder.getInt64(C: ~0xff000000ULL));
4910 Value *ShiftedScale = Builder.CreateShl(LHS: Scale, RHS: Builder.getInt64(C: 24));
4911
4912 return Builder.CreateOr(LHS: MaskedFPM, RHS: ShiftedScale);
4913 }
4914 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_lscale2) {
4915 Value *FPM = EmitScalarExpr(E: E->getArg(Arg: 0));
4916 Value *Scale = EmitScalarExpr(E: E->getArg(Arg: 1));
4917 Scale = Builder.CreateAnd(LHS: Scale, RHS: Builder.getInt64(C: 0x3f));
4918
4919 Value *LowFPM = Builder.CreateAnd(LHS: FPM, RHS: Builder.getInt64(C: ~0x3f00000000ULL));
4920 Value *ShiftedScale = Builder.CreateShl(LHS: Scale, RHS: Builder.getInt64(C: 32));
4921
4922 return Builder.CreateOr(LHS: LowFPM, RHS: ShiftedScale);
4923 }
4924
4925 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
4926 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
4927 Value *Pointer = EmitScalarExpr(E: E->getArg(Arg: 0));
4928 Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 1));
4929 assert(Mask->getType()->getScalarSizeInBits() == 64 &&
4930 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
4931 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
4932 Args: {Pointer, Mask});
4933 }
4934 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
4935 Value *Pointer = EmitScalarExpr(E: E->getArg(Arg: 0));
4936 Value *TagOffset = EmitScalarExpr(E: E->getArg(Arg: 1));
4937
4938 TagOffset = Builder.CreateZExt(V: TagOffset, DestTy: Int64Ty);
4939 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
4940 Args: {Pointer, TagOffset});
4941 }
4942 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
4943 Value *Pointer = EmitScalarExpr(E: E->getArg(Arg: 0));
4944 Value *ExcludedMask = EmitScalarExpr(E: E->getArg(Arg: 1));
4945 assert(ExcludedMask->getType()->getScalarSizeInBits() == 64 &&
4946 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
4947 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
4948 Args: {Pointer, ExcludedMask});
4949 }
4950 // Although it is possible to supply a different return
4951 // address (first arg) to this intrinsic, for now we set
4952 // return address same as input address.
4953 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
4954 Value *TagAddress = EmitScalarExpr(E: E->getArg(Arg: 0));
4955 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
4956 Args: {TagAddress, TagAddress});
4957 }
4958 // Although it is possible to supply a different tag (to set)
4959 // to this intrinsic (as first arg), for now we supply
4960 // the tag that is in input address arg (common use case).
4961 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
4962 Value *TagAddress = EmitScalarExpr(E: E->getArg(Arg: 0));
4963 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
4964 Args: {TagAddress, TagAddress});
4965 }
4966 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
4967 Value *PointerA = EmitScalarExpr(E: E->getArg(Arg: 0));
4968 Value *PointerB = EmitScalarExpr(E: E->getArg(Arg: 1));
4969 return Builder.CreateCall(
4970 Callee: CGM.getIntrinsic(IID: MTEIntrinsicID), Args: {PointerA, PointerB});
4971 }
4972 }
4973
4974 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
4975 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
4976 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
4977 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
4978 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
4979 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
4980 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
4981 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
4982
4983 SpecialRegisterAccessKind AccessKind = Write;
4984 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
4985 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
4986 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
4987 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
4988 AccessKind = VolatileRead;
4989
4990 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
4991 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
4992
4993 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
4994 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
4995
4996 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
4997 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
4998
4999 llvm::Type *ValueType;
5000 llvm::Type *RegisterType = Int64Ty;
5001 if (Is32Bit) {
5002 ValueType = Int32Ty;
5003 } else if (Is128Bit) {
5004 llvm::Type *Int128Ty =
5005 llvm::IntegerType::getInt128Ty(C&: CGM.getLLVMContext());
5006 ValueType = Int128Ty;
5007 RegisterType = Int128Ty;
5008 } else if (IsPointerBuiltin) {
5009 ValueType = VoidPtrTy;
5010 } else {
5011 ValueType = Int64Ty;
5012 };
5013
5014 return EmitSpecialRegisterBuiltin(CGF&: *this, E, RegisterType, ValueType,
5015 AccessKind);
5016 }
5017
5018 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
5019 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
5020 LLVMContext &Context = CGM.getLLVMContext();
5021
5022 unsigned SysReg =
5023 E->getArg(Arg: 0)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
5024
5025 std::string SysRegStr;
5026 llvm::raw_string_ostream(SysRegStr)
5027 << (0b10 | SysReg >> 14) << ":" << ((SysReg >> 11) & 7) << ":"
5028 << ((SysReg >> 7) & 15) << ":" << ((SysReg >> 3) & 15) << ":"
5029 << (SysReg & 7);
5030
5031 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, Str: SysRegStr) };
5032 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
5033 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
5034
5035 llvm::Type *RegisterType = Int64Ty;
5036 llvm::Type *Types[] = { RegisterType };
5037
5038 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
5039 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::read_register, Tys: Types);
5040
5041 return Builder.CreateCall(Callee: F, Args: Metadata);
5042 }
5043
5044 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::write_register, Tys: Types);
5045 llvm::Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 1));
5046 llvm::Value *Result = Builder.CreateCall(Callee: F, Args: {Metadata, ArgValue});
5047
5048 return Result;
5049 }
5050
5051 if (BuiltinID == clang::AArch64::BI__sys) {
5052 unsigned SysReg =
5053 E->getArg(Arg: 0)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
5054 const unsigned Op1 = SysReg >> 11;
5055 const unsigned CRn = (SysReg >> 7) & 0xf;
5056 const unsigned CRm = (SysReg >> 3) & 0xf;
5057 const unsigned Op2 = SysReg & 0x7;
5058
5059 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_sys),
5060 Args: {Builder.getInt32(C: Op1), Builder.getInt32(C: CRn),
5061 Builder.getInt32(C: CRm), Builder.getInt32(C: Op2),
5062 EmitScalarExpr(E: E->getArg(Arg: 1))});
5063
5064 // Return 0 for convenience, even though MSVC returns some other undefined
5065 // value.
5066 return ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
5067 }
5068
5069 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
5070 llvm::Function *F =
5071 CGM.getIntrinsic(IID: Intrinsic::addressofreturnaddress, Tys: AllocaInt8PtrTy);
5072 return Builder.CreateCall(Callee: F);
5073 }
5074
5075 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
5076 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: AllocaInt8PtrTy);
5077 return Builder.CreateCall(Callee: F);
5078 }
5079
5080 if (BuiltinID == clang::AArch64::BI__mulh ||
5081 BuiltinID == clang::AArch64::BI__umulh) {
5082 llvm::Type *ResType = ConvertType(T: E->getType());
5083 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 128);
5084
5085 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
5086 Value *LHS =
5087 Builder.CreateIntCast(V: EmitScalarExpr(E: E->getArg(Arg: 0)), DestTy: Int128Ty, isSigned: IsSigned);
5088 Value *RHS =
5089 Builder.CreateIntCast(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: Int128Ty, isSigned: IsSigned);
5090
5091 Value *MulResult, *HigherBits;
5092 if (IsSigned) {
5093 MulResult = Builder.CreateNSWMul(LHS, RHS);
5094 HigherBits = Builder.CreateAShr(LHS: MulResult, RHS: 64);
5095 } else {
5096 MulResult = Builder.CreateNUWMul(LHS, RHS);
5097 HigherBits = Builder.CreateLShr(LHS: MulResult, RHS: 64);
5098 }
5099 HigherBits = Builder.CreateIntCast(V: HigherBits, DestTy: ResType, isSigned: IsSigned);
5100
5101 return HigherBits;
5102 }
5103
5104 if (BuiltinID == AArch64::BI__writex18byte ||
5105 BuiltinID == AArch64::BI__writex18word ||
5106 BuiltinID == AArch64::BI__writex18dword ||
5107 BuiltinID == AArch64::BI__writex18qword) {
5108 // Process the args first
5109 Value *OffsetArg = EmitScalarExpr(E: E->getArg(Arg: 0));
5110 Value *DataArg = EmitScalarExpr(E: E->getArg(Arg: 1));
5111
5112 // Read x18 as i8*
5113 llvm::Value *X18 = readX18AsPtr(CGF&: *this);
5114
5115 // Store val at x18 + offset
5116 Value *Offset = Builder.CreateZExt(V: OffsetArg, DestTy: Int64Ty);
5117 Value *Ptr = Builder.CreateGEP(Ty: Int8Ty, Ptr: X18, IdxList: Offset);
5118 StoreInst *Store =
5119 Builder.CreateAlignedStore(Val: DataArg, Addr: Ptr, Align: CharUnits::One());
5120 return Store;
5121 }
5122
5123 if (BuiltinID == AArch64::BI__readx18byte ||
5124 BuiltinID == AArch64::BI__readx18word ||
5125 BuiltinID == AArch64::BI__readx18dword ||
5126 BuiltinID == AArch64::BI__readx18qword) {
5127 // Process the args first
5128 Value *OffsetArg = EmitScalarExpr(E: E->getArg(Arg: 0));
5129
5130 // Read x18 as i8*
5131 llvm::Value *X18 = readX18AsPtr(CGF&: *this);
5132
5133 // Load x18 + offset
5134 Value *Offset = Builder.CreateZExt(V: OffsetArg, DestTy: Int64Ty);
5135 Value *Ptr = Builder.CreateGEP(Ty: Int8Ty, Ptr: X18, IdxList: Offset);
5136 llvm::Type *IntTy = ConvertType(T: E->getType());
5137 LoadInst *Load = Builder.CreateAlignedLoad(Ty: IntTy, Addr: Ptr, Align: CharUnits::One());
5138 return Load;
5139 }
5140
5141 if (BuiltinID == AArch64::BI__addx18byte ||
5142 BuiltinID == AArch64::BI__addx18word ||
5143 BuiltinID == AArch64::BI__addx18dword ||
5144 BuiltinID == AArch64::BI__addx18qword ||
5145 BuiltinID == AArch64::BI__incx18byte ||
5146 BuiltinID == AArch64::BI__incx18word ||
5147 BuiltinID == AArch64::BI__incx18dword ||
5148 BuiltinID == AArch64::BI__incx18qword) {
5149 llvm::Type *IntTy;
5150 bool isIncrement;
5151 switch (BuiltinID) {
5152 case AArch64::BI__incx18byte:
5153 IntTy = Int8Ty;
5154 isIncrement = true;
5155 break;
5156 case AArch64::BI__incx18word:
5157 IntTy = Int16Ty;
5158 isIncrement = true;
5159 break;
5160 case AArch64::BI__incx18dword:
5161 IntTy = Int32Ty;
5162 isIncrement = true;
5163 break;
5164 case AArch64::BI__incx18qword:
5165 IntTy = Int64Ty;
5166 isIncrement = true;
5167 break;
5168 default:
5169 IntTy = ConvertType(T: E->getArg(Arg: 1)->getType());
5170 isIncrement = false;
5171 break;
5172 }
5173 // Process the args first
5174 Value *OffsetArg = EmitScalarExpr(E: E->getArg(Arg: 0));
5175 Value *ValToAdd =
5176 isIncrement ? ConstantInt::get(Ty: IntTy, V: 1) : EmitScalarExpr(E: E->getArg(Arg: 1));
5177
5178 // Read x18 as i8*
5179 llvm::Value *X18 = readX18AsPtr(CGF&: *this);
5180
5181 // Load x18 + offset
5182 Value *Offset = Builder.CreateZExt(V: OffsetArg, DestTy: Int64Ty);
5183 Value *Ptr = Builder.CreateGEP(Ty: Int8Ty, Ptr: X18, IdxList: Offset);
5184 LoadInst *Load = Builder.CreateAlignedLoad(Ty: IntTy, Addr: Ptr, Align: CharUnits::One());
5185
5186 // Add values
5187 Value *AddResult = Builder.CreateAdd(LHS: Load, RHS: ValToAdd);
5188
5189 // Store val at x18 + offset
5190 StoreInst *Store =
5191 Builder.CreateAlignedStore(Val: AddResult, Addr: Ptr, Align: CharUnits::One());
5192 return Store;
5193 }
5194
5195 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
5196 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
5197 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
5198 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
5199 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
5200 llvm::Type *RetTy = ConvertType(T: E->getType());
5201 return Builder.CreateBitCast(V: Arg, DestTy: RetTy);
5202 }
5203
5204 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5205 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5206 BuiltinID == AArch64::BI_CountLeadingZeros ||
5207 BuiltinID == AArch64::BI_CountLeadingZeros64) {
5208 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
5209 llvm::Type *ArgType = Arg->getType();
5210
5211 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5212 BuiltinID == AArch64::BI_CountLeadingOnes64)
5213 Arg = Builder.CreateXor(LHS: Arg, RHS: Constant::getAllOnesValue(Ty: ArgType));
5214
5215 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
5216 Value *Result = Builder.CreateCall(Callee: F, Args: {Arg, Builder.getInt1(V: false)});
5217
5218 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5219 BuiltinID == AArch64::BI_CountLeadingZeros64)
5220 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5221 return Result;
5222 }
5223
5224 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
5225 BuiltinID == AArch64::BI_CountLeadingSigns64) {
5226 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
5227
5228 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
5229 ? CGM.getIntrinsic(IID: Intrinsic::aarch64_cls)
5230 : CGM.getIntrinsic(IID: Intrinsic::aarch64_cls64);
5231
5232 Value *Result = Builder.CreateCall(Callee: F, Args: Arg, Name: "cls");
5233 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
5234 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5235 return Result;
5236 }
5237
5238 if (BuiltinID == AArch64::BI_CountOneBits ||
5239 BuiltinID == AArch64::BI_CountOneBits64) {
5240 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
5241 llvm::Type *ArgType = ArgValue->getType();
5242 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
5243
5244 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
5245 if (BuiltinID == AArch64::BI_CountOneBits64)
5246 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5247 return Result;
5248 }
5249
5250 if (BuiltinID == AArch64::BI_CountTrailingZeros ||
5251 BuiltinID == AArch64::BI_CountTrailingZeros64) {
5252 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
5253 llvm::Type *ArgType = ArgValue->getType();
5254 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
5255
5256 // MSVC leaves 0 undefined; use false for predictable codegen
5257 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getInt1(V: false)});
5258 if (BuiltinID == AArch64::BI_CountTrailingZeros64)
5259 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5260 return Result;
5261 }
5262
5263 if (BuiltinID == AArch64::BI__prefetch) {
5264 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5265 Value *RW = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
5266 Value *Locality = ConstantInt::get(Ty: Int32Ty, V: 3);
5267 Value *Data = llvm::ConstantInt::get(Ty: Int32Ty, V: 1);
5268 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
5269 return Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, Data});
5270 }
5271
5272 if (BuiltinID == AArch64::BI__prefetch2) {
5273 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5274 llvm::APSInt PrfOp = E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: CGM.getContext());
5275 // Decode 5-bit PRFM encoding: bits[4:3]=type, bits[2:1]=target,
5276 // bit[0]=policy
5277 // type: PLD=0(load), PLI=1(instr), PST=2(store)
5278 // target: L1=0, L2=1, L3=2
5279 // policy: KEEP=0, STRM=1
5280 uint64_t Op = PrfOp.getZExtValue();
5281 uint64_t Type = (Op >> 3) & 0x3;
5282 uint64_t Target = (Op >> 1) & 0x3;
5283 uint64_t Policy = Op & 0x1;
5284 Value *RW = Builder.getInt32(C: Type == 2 ? 1 : 0);
5285 Value *Local = Builder.getInt32(C: Target);
5286 Value *IsStream = Builder.getInt32(C: Policy);
5287 Value *IsData = Builder.getInt32(C: Type == 1 ? 0 : 1);
5288 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_prefetch);
5289 return Builder.CreateCall(Callee: F, Args: {Address, RW, Local, IsStream, IsData});
5290 }
5291
5292 if (BuiltinID == AArch64::BI__hlt) {
5293 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_hlt);
5294 Builder.CreateCall(Callee: F, Args: {EmitScalarExpr(E: E->getArg(Arg: 0))});
5295
5296 // FIXME: MSVC documents __hlt as taking further arguments in X0-X3 and
5297 // returning the value in X0, like __hvc/__svc below. This ignores the
5298 // extra arguments and returns 0.
5299 return ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
5300 }
5301
5302 if (BuiltinID == AArch64::BI__hvc || BuiltinID == AArch64::BI__svc) {
5303 unsigned IID = BuiltinID == AArch64::BI__svc ? Intrinsic::aarch64_svc
5304 : Intrinsic::aarch64_hvc;
5305 // The first argument is the instruction immediate; it must be a constant
5306 // (ImmArg on the intrinsic, encoded in the instruction). The remaining
5307 // arguments (at most four, enforced by Sema) are passed in X0-X3, widened
5308 // to 64 bits. The intrinsic takes exactly four register operands, so any
5309 // unused trailing ones are passed as poison and dropped during lowering.
5310 SmallVector<Value *, 5> Args{Builder.getInt32(
5311 C: GetIntegerConstantValue<uint32_t>(E: E->getArg(Arg: 0), Context&: getContext()))};
5312 for (unsigned I = 1, N = E->getNumArgs(); I < N; ++I) {
5313 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: I));
5314 llvm::Type *ArgTy = Arg->getType();
5315 if (ArgTy->isPointerTy())
5316 Arg = Builder.CreatePtrToInt(V: Arg, DestTy: Int64Ty);
5317 else if (ArgTy->isFloatingPointTy())
5318 // Reinterpret the bits into the integer register, matching MSVC (e.g.
5319 // "fmov x0, d0" for a double).
5320 Arg = Builder.CreateZExtOrTrunc(
5321 V: Builder.CreateBitCast(
5322 V: Arg, DestTy: Builder.getIntNTy(N: ArgTy->getPrimitiveSizeInBits())),
5323 DestTy: Int64Ty);
5324 else
5325 Arg = Builder.CreateIntCast(
5326 V: Arg, DestTy: Int64Ty, isSigned: E->getArg(Arg: I)->getType()->isSignedIntegerType());
5327 Args.push_back(Elt: Arg);
5328 }
5329 while (Args.size() < 5)
5330 Args.push_back(Elt: llvm::PoisonValue::get(T: Int64Ty));
5331 Value *Call = Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args);
5332 // MSVC returns unsigned int, i.e. the low 32 bits of the X0 result.
5333 return Builder.CreateTrunc(V: Call, DestTy: Int32Ty);
5334 }
5335
5336 if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
5337 return Builder.CreateFPTrunc(
5338 V: Builder.CreateBitCast(V: EmitScalarExpr(E: E->getArg(Arg: 0)),
5339 DestTy: Builder.getFloatTy()),
5340 DestTy: Builder.getBFloatTy());
5341
5342 // Handle MSVC intrinsics before argument evaluation to prevent double
5343 // evaluation.
5344 if (std::optional<MSVCIntrin> MsvcIntId =
5345 translateAarch64ToMsvcIntrin(BuiltinID))
5346 return EmitMSVCBuiltinExpr(BuiltinID: *MsvcIntId, E);
5347
5348 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
5349 auto It = llvm::find_if(Range: NEONEquivalentIntrinsicMap, P: [BuiltinID](auto &P) {
5350 return P.first == BuiltinID;
5351 });
5352 if (It != end(arr: NEONEquivalentIntrinsicMap))
5353 BuiltinID = It->second;
5354
5355 // Check whether this is an SISD builtin.
5356 auto SISDMap = ArrayRef(AArch64SISDIntrinsicMap);
5357 const ARMNeonVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap(
5358 IntrinsicMap: SISDMap, BuiltinID, MapProvenSorted&: AArch64SISDIntrinsicsProvenSorted);
5359 bool IsSISD = (Builtin != nullptr);
5360
5361 // Find out if any arguments are required to be integer constant
5362 // expressions.
5363 unsigned ICEArguments = 0;
5364 ASTContext::GetBuiltinTypeError Error;
5365 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
5366 assert(Error == ASTContext::GE_None && "Should not codegen an error");
5367
5368 llvm::SmallVector<Value*, 4> Ops;
5369 Address PtrOp0 = Address::invalid();
5370 // Note the assumption that SISD intrinsics do not contain extra arguments.
5371 // TODO: Fold this into a single function call instead of, effectively, two
5372 // separate checks.
5373 bool HasExtraArg = !IsSISD && CodeGenUtils::hasExtraNeonArgument(BuiltinID);
5374 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
5375 for (unsigned i = 0, e = NumArgs; i != e; i++) {
5376 if (i == 0) {
5377 switch (BuiltinID) {
5378 case NEON::BI__builtin_neon_vld1_v:
5379 case NEON::BI__builtin_neon_vld1q_v:
5380 case NEON::BI__builtin_neon_vld1_dup_v:
5381 case NEON::BI__builtin_neon_vld1q_dup_v:
5382 case NEON::BI__builtin_neon_vld1_lane_v:
5383 case NEON::BI__builtin_neon_vld1q_lane_v:
5384 case NEON::BI__builtin_neon_vst1_v:
5385 case NEON::BI__builtin_neon_vst1q_v:
5386 case NEON::BI__builtin_neon_vst1_lane_v:
5387 case NEON::BI__builtin_neon_vst1q_lane_v:
5388 case NEON::BI__builtin_neon_vldap1_lane_s64:
5389 case NEON::BI__builtin_neon_vldap1q_lane_s64:
5390 case NEON::BI__builtin_neon_vstl1_lane_s64:
5391 case NEON::BI__builtin_neon_vstl1q_lane_s64:
5392 // Get the alignment for the argument in addition to the value;
5393 // we'll use it later.
5394 PtrOp0 = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5395 Ops.push_back(Elt: PtrOp0.emitRawPointer(CGF&: *this));
5396 continue;
5397 }
5398 }
5399 Ops.push_back(Elt: EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E));
5400 }
5401
5402 if (Builtin) {
5403 Value *Result = EmitCommonNeonSISDBuiltinExpr(CGF&: *this, SISDInfo: *Builtin, Ops, E);
5404 assert(Result && "SISD intrinsic should have been handled");
5405 return Result;
5406 }
5407
5408 const Expr *Arg = E->getArg(Arg: E->getNumArgs()-1);
5409 NeonTypeFlags Type(0);
5410 if (std::optional<llvm::APSInt> Result =
5411 Arg->getIntegerConstantExpr(Ctx: getContext()))
5412 // Determine the type of this overloaded NEON intrinsic.
5413 Type = NeonTypeFlags(Result->getZExtValue());
5414
5415 bool usgn = Type.isUnsigned();
5416 bool quad = Type.isQuad();
5417 unsigned Int;
5418
5419 // Not all intrinsics handled by the common case work for AArch64 yet, so only
5420 // defer to common code if it's been added to our special map.
5421 Builtin =
5422 findARMVectorIntrinsicInMap(IntrinsicMap: ArrayRef(AArch64SIMDIntrinsicMap), BuiltinID,
5423 MapProvenSorted&: AArch64SIMDIntrinsicsProvenSorted);
5424
5425 if (Builtin)
5426 return EmitCommonNeonBuiltinExpr(
5427 BuiltinID: Builtin->BuiltinID, LLVMIntrinsic: Builtin->LLVMIntrinsic, AltLLVMIntrinsic: Builtin->AltLLVMIntrinsic,
5428 NameHint: Builtin->NameHint, Modifier: Builtin->TypeModifier, E, Ops,
5429 /*never use addresses*/ PtrOp0: Address::invalid(), PtrOp1: Address::invalid(), Arch);
5430
5431 if (Value *V = EmitAArch64TblBuiltinExpr(CGF&: *this, BuiltinID, E, Ops, Arch))
5432 return V;
5433
5434 // Handle non-overloaded intrinsics first.
5435 switch (BuiltinID) {
5436 default: break;
5437 case NEON::BI__builtin_neon_vabsh_f16:
5438 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::fabs, Tys: HalfTy), Ops, name: "vabs");
5439 case NEON::BI__builtin_neon_vaddq_p128: {
5440 llvm::Type *Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags::Poly128);
5441 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
5442 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
5443 Ops[0] = Builder.CreateXor(LHS: Ops[0], RHS: Ops[1]);
5444 llvm::Type *Int128Ty = llvm::Type::getIntNTy(C&: getLLVMContext(), N: 128);
5445 return Builder.CreateBitCast(V: Ops[0], DestTy: Int128Ty);
5446 }
5447 case NEON::BI__builtin_neon_vldrq_p128: {
5448 llvm::Type *Int128Ty = llvm::Type::getIntNTy(C&: getLLVMContext(), N: 128);
5449 return Builder.CreateAlignedLoad(Ty: Int128Ty, Addr: Ops[0],
5450 Align: CharUnits::fromQuantity(Quantity: 16));
5451 }
5452 case NEON::BI__builtin_neon_vstrq_p128: {
5453 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
5454 }
5455 case NEON::BI__builtin_neon_vcvts_f32_u32:
5456 case NEON::BI__builtin_neon_vcvtd_f64_u64:
5457 usgn = true;
5458 [[fallthrough]];
5459 case NEON::BI__builtin_neon_vcvts_f32_s32:
5460 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5461 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5462 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
5463 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
5464 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: InTy);
5465 if (usgn)
5466 return Builder.CreateUIToFP(V: Ops[0], DestTy: FTy);
5467 return Builder.CreateSIToFP(V: Ops[0], DestTy: FTy);
5468 }
5469 case NEON::BI__builtin_neon_vcvth_f16_u16:
5470 case NEON::BI__builtin_neon_vcvth_f16_u32:
5471 case NEON::BI__builtin_neon_vcvth_f16_u64:
5472 usgn = true;
5473 [[fallthrough]];
5474 case NEON::BI__builtin_neon_vcvth_f16_s16:
5475 case NEON::BI__builtin_neon_vcvth_f16_s32:
5476 case NEON::BI__builtin_neon_vcvth_f16_s64: {
5477 llvm::Type *FTy = HalfTy;
5478 llvm::Type *InTy;
5479 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
5480 InTy = Int64Ty;
5481 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
5482 InTy = Int32Ty;
5483 else
5484 InTy = Int16Ty;
5485 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: InTy);
5486 if (usgn)
5487 return Builder.CreateUIToFP(V: Ops[0], DestTy: FTy);
5488 return Builder.CreateSIToFP(V: Ops[0], DestTy: FTy);
5489 }
5490 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5491 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5492 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5493 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5494 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5495 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5496 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5497 case NEON::BI__builtin_neon_vcvtph_s16_f16: {
5498 llvm::Type *InTy = Int16Ty;
5499 llvm::Type* FTy = HalfTy;
5500 llvm::Type *Tys[2] = {InTy, FTy};
5501 switch (BuiltinID) {
5502 default: llvm_unreachable("missing builtin ID in switch!");
5503 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5504 Int = Intrinsic::aarch64_neon_fcvtau; break;
5505 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5506 Int = Intrinsic::aarch64_neon_fcvtmu; break;
5507 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5508 Int = Intrinsic::aarch64_neon_fcvtnu; break;
5509 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5510 Int = Intrinsic::aarch64_neon_fcvtpu; break;
5511 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5512 Int = Intrinsic::aarch64_neon_fcvtas; break;
5513 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5514 Int = Intrinsic::aarch64_neon_fcvtms; break;
5515 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5516 Int = Intrinsic::aarch64_neon_fcvtns; break;
5517 case NEON::BI__builtin_neon_vcvtph_s16_f16:
5518 Int = Intrinsic::aarch64_neon_fcvtps; break;
5519 }
5520 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fcvt");
5521 }
5522 case NEON::BI__builtin_neon_vcaleh_f16:
5523 case NEON::BI__builtin_neon_vcalth_f16:
5524 case NEON::BI__builtin_neon_vcageh_f16:
5525 case NEON::BI__builtin_neon_vcagth_f16: {
5526 llvm::Type* InTy = Int32Ty;
5527 llvm::Type* FTy = HalfTy;
5528 llvm::Type *Tys[2] = {InTy, FTy};
5529 switch (BuiltinID) {
5530 default: llvm_unreachable("missing builtin ID in switch!");
5531 case NEON::BI__builtin_neon_vcageh_f16:
5532 Int = Intrinsic::aarch64_neon_facge; break;
5533 case NEON::BI__builtin_neon_vcagth_f16:
5534 Int = Intrinsic::aarch64_neon_facgt; break;
5535 case NEON::BI__builtin_neon_vcaleh_f16:
5536 Int = Intrinsic::aarch64_neon_facge; std::swap(a&: Ops[0], b&: Ops[1]); break;
5537 case NEON::BI__builtin_neon_vcalth_f16:
5538 Int = Intrinsic::aarch64_neon_facgt; std::swap(a&: Ops[0], b&: Ops[1]); break;
5539 }
5540 Ops[0] = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "facg");
5541 return Builder.CreateTrunc(V: Ops[0], DestTy: Int16Ty);
5542 }
5543 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5544 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
5545 llvm::Type* InTy = Int32Ty;
5546 llvm::Type* FTy = HalfTy;
5547 llvm::Type *Tys[2] = {InTy, FTy};
5548 switch (BuiltinID) {
5549 default: llvm_unreachable("missing builtin ID in switch!");
5550 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5551 Int = Intrinsic::aarch64_neon_vcvtfp2fxs; break;
5552 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
5553 Int = Intrinsic::aarch64_neon_vcvtfp2fxu; break;
5554 }
5555 Ops[0] = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fcvth_n");
5556 return Builder.CreateTrunc(V: Ops[0], DestTy: Int16Ty);
5557 }
5558 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5559 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
5560 llvm::Type* FTy = HalfTy;
5561 llvm::Type* InTy = Int32Ty;
5562 llvm::Type *Tys[2] = {FTy, InTy};
5563 switch (BuiltinID) {
5564 default: llvm_unreachable("missing builtin ID in switch!");
5565 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5566 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
5567 Ops[0] = Builder.CreateSExt(V: Ops[0], DestTy: InTy, Name: "sext");
5568 break;
5569 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
5570 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
5571 Ops[0] = Builder.CreateZExt(V: Ops[0], DestTy: InTy);
5572 break;
5573 }
5574 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fcvth_n");
5575 }
5576 case NEON::BI__builtin_neon_vpaddd_s64: {
5577 // TODO: Isn't this handled by
5578 // EmitCommonNeonSISDBuiltinExpr?
5579 auto *Ty = llvm::FixedVectorType::get(ElementType: Int64Ty, NumElts: 2);
5580 // The vector is v2f64, so make sure it's bitcast to that.
5581 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty, Name: "v2i64");
5582 llvm::Value *Idx0 = llvm::ConstantInt::get(Ty: SizeTy, V: 0);
5583 llvm::Value *Idx1 = llvm::ConstantInt::get(Ty: SizeTy, V: 1);
5584 Value *Op0 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx0, Name: "lane0");
5585 Value *Op1 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx1, Name: "lane1");
5586 // Pairwise addition of a v2f64 into a scalar f64.
5587 return Builder.CreateAdd(LHS: Op0, RHS: Op1, Name: "vpaddd");
5588 }
5589 case NEON::BI__builtin_neon_vpaddd_f64: {
5590 auto *Ty = llvm::FixedVectorType::get(ElementType: DoubleTy, NumElts: 2);
5591 // The vector is v2f64, so make sure it's bitcast to that.
5592 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty, Name: "v2f64");
5593 llvm::Value *Idx0 = llvm::ConstantInt::get(Ty: SizeTy, V: 0);
5594 llvm::Value *Idx1 = llvm::ConstantInt::get(Ty: SizeTy, V: 1);
5595 Value *Op0 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx0, Name: "lane0");
5596 Value *Op1 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx1, Name: "lane1");
5597 // Pairwise addition of a v2f64 into a scalar f64.
5598 return Builder.CreateFAdd(L: Op0, R: Op1, Name: "vpaddd");
5599 }
5600 case NEON::BI__builtin_neon_vpadds_f32: {
5601 auto *Ty = llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 2);
5602 // The vector is v2f32, so make sure it's bitcast to that.
5603 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty, Name: "v2f32");
5604 llvm::Value *Idx0 = llvm::ConstantInt::get(Ty: SizeTy, V: 0);
5605 llvm::Value *Idx1 = llvm::ConstantInt::get(Ty: SizeTy, V: 1);
5606 Value *Op0 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx0, Name: "lane0");
5607 Value *Op1 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx1, Name: "lane1");
5608 // Pairwise addition of a v2f32 into a scalar f32.
5609 return Builder.CreateFAdd(L: Op0, R: Op1, Name: "vpaddd");
5610 }
5611 case NEON::BI__builtin_neon_vceqzd_s64:
5612 return EmitAArch64CompareBuiltinExpr(
5613 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5614 Pred: ICmpInst::ICMP_EQ, Name: "vceqz");
5615 case NEON::BI__builtin_neon_vceqzd_f64:
5616 case NEON::BI__builtin_neon_vceqzs_f32:
5617 case NEON::BI__builtin_neon_vceqzh_f16:
5618 return EmitAArch64CompareBuiltinExpr(
5619 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5620 Pred: ICmpInst::FCMP_OEQ, Name: "vceqz");
5621 case NEON::BI__builtin_neon_vcgezd_s64:
5622 return EmitAArch64CompareBuiltinExpr(
5623 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5624 Pred: ICmpInst::ICMP_SGE, Name: "vcgez");
5625 case NEON::BI__builtin_neon_vcgezd_f64:
5626 case NEON::BI__builtin_neon_vcgezs_f32:
5627 case NEON::BI__builtin_neon_vcgezh_f16:
5628 return EmitAArch64CompareBuiltinExpr(
5629 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5630 Pred: ICmpInst::FCMP_OGE, Name: "vcgez");
5631 case NEON::BI__builtin_neon_vclezd_s64:
5632 return EmitAArch64CompareBuiltinExpr(
5633 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5634 Pred: ICmpInst::ICMP_SLE, Name: "vclez");
5635 case NEON::BI__builtin_neon_vclezd_f64:
5636 case NEON::BI__builtin_neon_vclezs_f32:
5637 case NEON::BI__builtin_neon_vclezh_f16:
5638 return EmitAArch64CompareBuiltinExpr(
5639 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5640 Pred: ICmpInst::FCMP_OLE, Name: "vclez");
5641 case NEON::BI__builtin_neon_vcgtzd_s64:
5642 return EmitAArch64CompareBuiltinExpr(
5643 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5644 Pred: ICmpInst::ICMP_SGT, Name: "vcgtz");
5645 case NEON::BI__builtin_neon_vcgtzd_f64:
5646 case NEON::BI__builtin_neon_vcgtzs_f32:
5647 case NEON::BI__builtin_neon_vcgtzh_f16:
5648 return EmitAArch64CompareBuiltinExpr(
5649 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5650 Pred: ICmpInst::FCMP_OGT, Name: "vcgtz");
5651 case NEON::BI__builtin_neon_vcltzd_s64:
5652 return EmitAArch64CompareBuiltinExpr(
5653 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5654 Pred: ICmpInst::ICMP_SLT, Name: "vcltz");
5655
5656 case NEON::BI__builtin_neon_vcltzd_f64:
5657 case NEON::BI__builtin_neon_vcltzs_f32:
5658 case NEON::BI__builtin_neon_vcltzh_f16:
5659 return EmitAArch64CompareBuiltinExpr(
5660 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5661 Pred: ICmpInst::FCMP_OLT, Name: "vcltz");
5662
5663 case NEON::BI__builtin_neon_vceqzd_u64: {
5664 return EmitAArch64CompareBuiltinExpr(
5665 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5666 Pred: ICmpInst::ICMP_EQ, Name: "vceqzd");
5667 }
5668 case NEON::BI__builtin_neon_vceqd_f64:
5669 case NEON::BI__builtin_neon_vcled_f64:
5670 case NEON::BI__builtin_neon_vcltd_f64:
5671 case NEON::BI__builtin_neon_vcged_f64:
5672 case NEON::BI__builtin_neon_vcgtd_f64: {
5673 llvm::CmpInst::Predicate P;
5674 switch (BuiltinID) {
5675 default: llvm_unreachable("missing builtin ID in switch!");
5676 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
5677 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
5678 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
5679 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
5680 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
5681 }
5682 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
5683 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: DoubleTy);
5684 if (P == llvm::FCmpInst::FCMP_OEQ)
5685 Ops[0] = Builder.CreateFCmp(P, LHS: Ops[0], RHS: Ops[1]);
5686 else
5687 Ops[0] = Builder.CreateFCmpS(P, LHS: Ops[0], RHS: Ops[1]);
5688 return Builder.CreateSExt(V: Ops[0], DestTy: Int64Ty, Name: "vcmpd");
5689 }
5690 case NEON::BI__builtin_neon_vceqs_f32:
5691 case NEON::BI__builtin_neon_vcles_f32:
5692 case NEON::BI__builtin_neon_vclts_f32:
5693 case NEON::BI__builtin_neon_vcges_f32:
5694 case NEON::BI__builtin_neon_vcgts_f32: {
5695 llvm::CmpInst::Predicate P;
5696 switch (BuiltinID) {
5697 default: llvm_unreachable("missing builtin ID in switch!");
5698 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
5699 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
5700 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
5701 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
5702 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
5703 }
5704 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: FloatTy);
5705 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: FloatTy);
5706 if (P == llvm::FCmpInst::FCMP_OEQ)
5707 Ops[0] = Builder.CreateFCmp(P, LHS: Ops[0], RHS: Ops[1]);
5708 else
5709 Ops[0] = Builder.CreateFCmpS(P, LHS: Ops[0], RHS: Ops[1]);
5710 return Builder.CreateSExt(V: Ops[0], DestTy: Int32Ty, Name: "vcmpd");
5711 }
5712 case NEON::BI__builtin_neon_vceqh_f16:
5713 case NEON::BI__builtin_neon_vcleh_f16:
5714 case NEON::BI__builtin_neon_vclth_f16:
5715 case NEON::BI__builtin_neon_vcgeh_f16:
5716 case NEON::BI__builtin_neon_vcgth_f16: {
5717 llvm::CmpInst::Predicate P;
5718 switch (BuiltinID) {
5719 default: llvm_unreachable("missing builtin ID in switch!");
5720 case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ; break;
5721 case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE; break;
5722 case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT; break;
5723 case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE; break;
5724 case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT; break;
5725 }
5726 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: HalfTy);
5727 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: HalfTy);
5728 if (P == llvm::FCmpInst::FCMP_OEQ)
5729 Ops[0] = Builder.CreateFCmp(P, LHS: Ops[0], RHS: Ops[1]);
5730 else
5731 Ops[0] = Builder.CreateFCmpS(P, LHS: Ops[0], RHS: Ops[1]);
5732 return Builder.CreateSExt(V: Ops[0], DestTy: Int16Ty, Name: "vcmpd");
5733 }
5734 case NEON::BI__builtin_neon_vceqd_s64:
5735 case NEON::BI__builtin_neon_vceqd_u64:
5736 case NEON::BI__builtin_neon_vcgtd_s64:
5737 case NEON::BI__builtin_neon_vcgtd_u64:
5738 case NEON::BI__builtin_neon_vcltd_s64:
5739 case NEON::BI__builtin_neon_vcltd_u64:
5740 case NEON::BI__builtin_neon_vcged_u64:
5741 case NEON::BI__builtin_neon_vcged_s64:
5742 case NEON::BI__builtin_neon_vcled_u64:
5743 case NEON::BI__builtin_neon_vcled_s64: {
5744 llvm::CmpInst::Predicate P;
5745 switch (BuiltinID) {
5746 default: llvm_unreachable("missing builtin ID in switch!");
5747 case NEON::BI__builtin_neon_vceqd_s64:
5748 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
5749 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
5750 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
5751 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
5752 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
5753 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
5754 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
5755 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
5756 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
5757 }
5758 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Int64Ty);
5759 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty);
5760 Ops[0] = Builder.CreateICmp(P, LHS: Ops[0], RHS: Ops[1]);
5761 return Builder.CreateSExt(V: Ops[0], DestTy: Int64Ty, Name: "vceqd");
5762 }
5763 case NEON::BI__builtin_neon_vnegd_s64:
5764 return Builder.CreateNeg(V: Ops[0], Name: "vnegd");
5765 case NEON::BI__builtin_neon_vnegh_f16:
5766 return Builder.CreateFNeg(V: Ops[0], Name: "vnegh");
5767 case NEON::BI__builtin_neon_vtstd_s64:
5768 case NEON::BI__builtin_neon_vtstd_u64: {
5769 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Int64Ty);
5770 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty);
5771 Ops[0] = Builder.CreateAnd(LHS: Ops[0], RHS: Ops[1]);
5772 Ops[0] = Builder.CreateICmp(P: ICmpInst::ICMP_NE, LHS: Ops[0],
5773 RHS: llvm::Constant::getNullValue(Ty: Int64Ty));
5774 return Builder.CreateSExt(V: Ops[0], DestTy: Int64Ty, Name: "vtstd");
5775 }
5776 case NEON::BI__builtin_neon_vset_lane_i8:
5777 case NEON::BI__builtin_neon_vset_lane_i16:
5778 case NEON::BI__builtin_neon_vset_lane_i32:
5779 case NEON::BI__builtin_neon_vset_lane_i64:
5780 case NEON::BI__builtin_neon_vset_lane_bf16:
5781 case NEON::BI__builtin_neon_vset_lane_f32:
5782 case NEON::BI__builtin_neon_vsetq_lane_i8:
5783 case NEON::BI__builtin_neon_vsetq_lane_i16:
5784 case NEON::BI__builtin_neon_vsetq_lane_i32:
5785 case NEON::BI__builtin_neon_vsetq_lane_i64:
5786 case NEON::BI__builtin_neon_vsetq_lane_bf16:
5787 case NEON::BI__builtin_neon_vsetq_lane_f32:
5788 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5789 case NEON::BI__builtin_neon_vset_lane_f64:
5790 // The vector type needs a cast for the v1f64 variant.
5791 Ops[1] =
5792 Builder.CreateBitCast(V: Ops[1], DestTy: llvm::FixedVectorType::get(ElementType: DoubleTy, NumElts: 1));
5793 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5794 case NEON::BI__builtin_neon_vset_lane_mf8:
5795 case NEON::BI__builtin_neon_vsetq_lane_mf8:
5796 // The input vector type needs a cast to scalar type.
5797 Ops[0] =
5798 Builder.CreateBitCast(V: Ops[0], DestTy: llvm::Type::getInt8Ty(C&: getLLVMContext()));
5799 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5800 case NEON::BI__builtin_neon_vsetq_lane_f64:
5801 // The vector type needs a cast for the v2f64 variant.
5802 Ops[1] =
5803 Builder.CreateBitCast(V: Ops[1], DestTy: llvm::FixedVectorType::get(ElementType: DoubleTy, NumElts: 2));
5804 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5805
5806 case NEON::BI__builtin_neon_vget_lane_i8:
5807 case NEON::BI__builtin_neon_vdupb_lane_i8:
5808 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5809 case NEON::BI__builtin_neon_vgetq_lane_i8:
5810 case NEON::BI__builtin_neon_vdupb_laneq_i8:
5811 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5812 case NEON::BI__builtin_neon_vget_lane_mf8:
5813 case NEON::BI__builtin_neon_vdupb_lane_mf8:
5814 case NEON::BI__builtin_neon_vgetq_lane_mf8:
5815 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
5816 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5817 case NEON::BI__builtin_neon_vget_lane_i16:
5818 case NEON::BI__builtin_neon_vduph_lane_i16:
5819 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5820 case NEON::BI__builtin_neon_vgetq_lane_i16:
5821 case NEON::BI__builtin_neon_vduph_laneq_i16:
5822 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5823 case NEON::BI__builtin_neon_vget_lane_i32:
5824 case NEON::BI__builtin_neon_vdups_lane_i32:
5825 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5826 case NEON::BI__builtin_neon_vdups_lane_f32:
5827 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vdups_lane");
5828 case NEON::BI__builtin_neon_vgetq_lane_i32:
5829 case NEON::BI__builtin_neon_vdups_laneq_i32:
5830 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5831 case NEON::BI__builtin_neon_vget_lane_i64:
5832 case NEON::BI__builtin_neon_vdupd_lane_i64:
5833 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5834 case NEON::BI__builtin_neon_vdupd_lane_f64:
5835 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vdupd_lane");
5836 case NEON::BI__builtin_neon_vgetq_lane_i64:
5837 case NEON::BI__builtin_neon_vdupd_laneq_i64:
5838 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5839 case NEON::BI__builtin_neon_vget_lane_f32:
5840 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5841 case NEON::BI__builtin_neon_vget_lane_f64:
5842 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5843 case NEON::BI__builtin_neon_vgetq_lane_f32:
5844 case NEON::BI__builtin_neon_vdups_laneq_f32:
5845 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5846 case NEON::BI__builtin_neon_vgetq_lane_f64:
5847 case NEON::BI__builtin_neon_vdupd_laneq_f64:
5848 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5849 case NEON::BI__builtin_neon_vaddh_f16:
5850 return Builder.CreateFAdd(L: Ops[0], R: Ops[1], Name: "vaddh");
5851 case NEON::BI__builtin_neon_vsubh_f16:
5852 return Builder.CreateFSub(L: Ops[0], R: Ops[1], Name: "vsubh");
5853 case NEON::BI__builtin_neon_vmulh_f16:
5854 return Builder.CreateFMul(L: Ops[0], R: Ops[1], Name: "vmulh");
5855 case NEON::BI__builtin_neon_vdivh_f16:
5856 return Builder.CreateFDiv(L: Ops[0], R: Ops[1], Name: "vdivh");
5857 case NEON::BI__builtin_neon_vfmah_f16:
5858 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5859 return emitCallMaybeConstrainedFPBuiltin(
5860 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty: HalfTy,
5861 Args: {Ops[1], Ops[2], Ops[0]});
5862 case NEON::BI__builtin_neon_vfmsh_f16: {
5863 Value *Neg = Builder.CreateFNeg(V: Ops[1], Name: "vsubh");
5864
5865 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5866 return emitCallMaybeConstrainedFPBuiltin(
5867 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty: HalfTy,
5868 Args: {Neg, Ops[2], Ops[0]});
5869 }
5870 case NEON::BI__builtin_neon_vaddd_s64:
5871 case NEON::BI__builtin_neon_vaddd_u64:
5872 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1], Name: "vaddd");
5873 case NEON::BI__builtin_neon_vsubd_s64:
5874 case NEON::BI__builtin_neon_vsubd_u64:
5875 return Builder.CreateSub(LHS: Ops[0], RHS: Ops[1], Name: "vsubd");
5876 case NEON::BI__builtin_neon_vqdmlalh_s16:
5877 case NEON::BI__builtin_neon_vqdmlslh_s16: {
5878 SmallVector<Value *, 2> ProductOps;
5879 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[1]));
5880 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[2]));
5881 auto *VTy = llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: 4);
5882 Ops[1] = EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmull, Tys: VTy),
5883 Ops&: ProductOps, name: "vqdmlXl");
5884 Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
5885 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: CI, Name: "lane0");
5886
5887 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
5888 ? Intrinsic::aarch64_neon_sqadd
5889 : Intrinsic::aarch64_neon_sqsub;
5890 // Drop the 2nd multiplication argument before the accumulation
5891 Ops.pop_back();
5892 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccumInt, Tys: Int32Ty), Ops, name: "vqdmlXl");
5893 }
5894 case NEON::BI__builtin_neon_vqshlud_n_s64: {
5895 Ops[1] = Builder.CreateZExt(V: Ops[1], DestTy: Int64Ty);
5896 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqshlu, Tys: Int64Ty),
5897 Ops, name: "vqshlu_n");
5898 }
5899 case NEON::BI__builtin_neon_vqshld_n_u64:
5900 case NEON::BI__builtin_neon_vqshld_n_s64: {
5901 Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
5902 ? Intrinsic::aarch64_neon_uqshl
5903 : Intrinsic::aarch64_neon_sqshl;
5904 Ops[1] = Builder.CreateZExt(V: Ops[1], DestTy: Int64Ty);
5905 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Int64Ty), Ops, name: "vqshl_n");
5906 }
5907 case NEON::BI__builtin_neon_vrshrd_n_u64:
5908 case NEON::BI__builtin_neon_vrshrd_n_s64: {
5909 Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
5910 ? Intrinsic::aarch64_neon_urshl
5911 : Intrinsic::aarch64_neon_srshl;
5912 int SV = cast<ConstantInt>(Val: Ops[1])->getSExtValue();
5913 Ops[1] = ConstantInt::get(Ty: Int64Ty, V: -SV);
5914 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Int64Ty), Ops, name: "vrshr_n");
5915 }
5916 case NEON::BI__builtin_neon_vrsrad_n_u64:
5917 case NEON::BI__builtin_neon_vrsrad_n_s64: {
5918 Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
5919 ? Intrinsic::aarch64_neon_urshl
5920 : Intrinsic::aarch64_neon_srshl;
5921 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty);
5922 Ops[2] = Builder.CreateNeg(V: Ops[2]);
5923 Ops[1] = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Int, Tys: Int64Ty),
5924 Args: {Ops[1], Builder.CreateSExt(V: Ops[2], DestTy: Int64Ty)});
5925 return Builder.CreateAdd(LHS: Ops[0], RHS: Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty));
5926 }
5927 case NEON::BI__builtin_neon_vshld_n_s64:
5928 case NEON::BI__builtin_neon_vshld_n_u64: {
5929 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[1]);
5930 return Builder.CreateShl(
5931 LHS: Ops[0], RHS: ConstantInt::get(Ty: Int64Ty, V: Amt->getZExtValue()), Name: "shld_n");
5932 }
5933 case NEON::BI__builtin_neon_vshrd_n_s64: {
5934 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[1]);
5935 return Builder.CreateAShr(
5936 LHS: Ops[0], RHS: ConstantInt::get(Ty: Int64Ty, V: std::min(a: static_cast<uint64_t>(63),
5937 b: Amt->getZExtValue())),
5938 Name: "shrd_n");
5939 }
5940 case NEON::BI__builtin_neon_vshrd_n_u64: {
5941 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[1]);
5942 uint64_t ShiftAmt = Amt->getZExtValue();
5943 // Right-shifting an unsigned value by its size yields 0.
5944 if (ShiftAmt == 64)
5945 return ConstantInt::get(Ty: Int64Ty, V: 0);
5946 return Builder.CreateLShr(LHS: Ops[0], RHS: ConstantInt::get(Ty: Int64Ty, V: ShiftAmt),
5947 Name: "shrd_n");
5948 }
5949 case NEON::BI__builtin_neon_vsrad_n_s64: {
5950 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[2]);
5951 Ops[1] = Builder.CreateAShr(
5952 LHS: Ops[1], RHS: ConstantInt::get(Ty: Int64Ty, V: std::min(a: static_cast<uint64_t>(63),
5953 b: Amt->getZExtValue())),
5954 Name: "shrd_n");
5955 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
5956 }
5957 case NEON::BI__builtin_neon_vsrad_n_u64: {
5958 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[2]);
5959 uint64_t ShiftAmt = Amt->getZExtValue();
5960 // Right-shifting an unsigned value by its size yields 0.
5961 // As Op + 0 = Op, return Ops[0] directly.
5962 if (ShiftAmt == 64)
5963 return Ops[0];
5964 Ops[1] = Builder.CreateLShr(LHS: Ops[1], RHS: ConstantInt::get(Ty: Int64Ty, V: ShiftAmt),
5965 Name: "shrd_n");
5966 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
5967 }
5968 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
5969 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
5970 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
5971 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
5972 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "lane");
5973 SmallVector<Value *, 2> ProductOps;
5974 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[1]));
5975 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[2]));
5976 auto *VTy = llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: 4);
5977 Ops[1] = EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmull, Tys: VTy),
5978 Ops&: ProductOps, name: "vqdmlXl");
5979 Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
5980 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: CI, Name: "lane0");
5981 // Drop lane-selection and the corresponding vector argument (these have
5982 // already been used)
5983 Ops.pop_back_n(NumItems: 2);
5984
5985 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
5986 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
5987 ? Intrinsic::aarch64_neon_sqadd
5988 : Intrinsic::aarch64_neon_sqsub;
5989 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccInt, Tys: Int32Ty), Ops, name: "vqdmlXl");
5990 }
5991 case NEON::BI__builtin_neon_vqdmlals_s32:
5992 case NEON::BI__builtin_neon_vqdmlsls_s32: {
5993 SmallVector<Value *, 2> ProductOps;
5994 ProductOps.push_back(Elt: Ops[1]);
5995 ProductOps.push_back(Elt: Ops[2]);
5996 Ops[1] =
5997 EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmulls_scalar),
5998 Ops&: ProductOps, name: "vqdmlXl");
5999
6000 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
6001 ? Intrinsic::aarch64_neon_sqadd
6002 : Intrinsic::aarch64_neon_sqsub;
6003 // Drop the 2nd multiplication argument before the accumulation
6004 Ops.pop_back();
6005 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccumInt, Tys: Int64Ty), Ops, name: "vqdmlXl");
6006 }
6007 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
6008 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
6009 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
6010 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
6011 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "lane");
6012 SmallVector<Value *, 2> ProductOps;
6013 ProductOps.push_back(Elt: Ops[1]);
6014 ProductOps.push_back(Elt: Ops[2]);
6015 Ops[1] =
6016 EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmulls_scalar),
6017 Ops&: ProductOps, name: "vqdmlXl");
6018 // Drop lane-selection and the corresponding vector argument (these have
6019 // already been used)
6020 Ops.pop_back_n(NumItems: 2);
6021
6022 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
6023 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
6024 ? Intrinsic::aarch64_neon_sqadd
6025 : Intrinsic::aarch64_neon_sqsub;
6026 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccInt, Tys: Int64Ty), Ops, name: "vqdmlXl");
6027 }
6028 case NEON::BI__builtin_neon_vget_lane_bf16:
6029 case NEON::BI__builtin_neon_vduph_lane_bf16:
6030 case NEON::BI__builtin_neon_vduph_lane_f16: {
6031 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
6032 }
6033 case NEON::BI__builtin_neon_vgetq_lane_bf16:
6034 case NEON::BI__builtin_neon_vduph_laneq_bf16:
6035 case NEON::BI__builtin_neon_vduph_laneq_f16: {
6036 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
6037 }
6038 case NEON::BI__builtin_neon_vcvt_bf16_f32: {
6039 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6040 llvm::Type *V4BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 4);
6041 return Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F32), DestTy: V4BF16);
6042 }
6043 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32: {
6044 SmallVector<int, 16> ConcatMask(8);
6045 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
6046 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6047 llvm::Type *V4BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 4);
6048 llvm::Value *Trunc =
6049 Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F32), DestTy: V4BF16);
6050 return Builder.CreateShuffleVector(
6051 V1: Trunc, V2: ConstantAggregateZero::get(Ty: V4BF16), Mask: ConcatMask);
6052 }
6053 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32: {
6054 SmallVector<int, 16> ConcatMask(8);
6055 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
6056 SmallVector<int, 16> LoMask(4);
6057 std::iota(first: LoMask.begin(), last: LoMask.end(), value: 0);
6058 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6059 llvm::Type *V4BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 4);
6060 llvm::Type *V8BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 8);
6061 llvm::Value *Inactive = Builder.CreateShuffleVector(
6062 V: Builder.CreateBitCast(V: Ops[0], DestTy: V8BF16), Mask: LoMask);
6063 llvm::Value *Trunc =
6064 Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[1], DestTy: V4F32), DestTy: V4BF16);
6065 return Builder.CreateShuffleVector(V1: Inactive, V2: Trunc, Mask: ConcatMask);
6066 }
6067 case NEON::BI__builtin_neon_vcvt_f16_f32: {
6068 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6069 llvm::Type *V4F16 = FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 4);
6070 return Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F32), DestTy: V4F16);
6071 }
6072 case NEON::BI__builtin_neon_vcvt_f32_f16: {
6073 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6074 llvm::Type *V4F16 = FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 4);
6075 return Builder.CreateFPExt(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F16), DestTy: V4F32);
6076 }
6077
6078 case clang::AArch64::BI_InterlockedAdd:
6079 case clang::AArch64::BI_InterlockedAdd_acq:
6080 case clang::AArch64::BI_InterlockedAdd_rel:
6081 case clang::AArch64::BI_InterlockedAdd_nf:
6082 case clang::AArch64::BI_InterlockedAdd64:
6083 case clang::AArch64::BI_InterlockedAdd64_acq:
6084 case clang::AArch64::BI_InterlockedAdd64_rel:
6085 case clang::AArch64::BI_InterlockedAdd64_nf: {
6086 Address DestAddr = CheckAtomicAlignment(CGF&: *this, E);
6087 Value *Val = Ops[1];
6088 llvm::AtomicOrdering Ordering;
6089 switch (BuiltinID) {
6090 case clang::AArch64::BI_InterlockedAdd:
6091 case clang::AArch64::BI_InterlockedAdd64:
6092 Ordering = llvm::AtomicOrdering::SequentiallyConsistent;
6093 break;
6094 case clang::AArch64::BI_InterlockedAdd_acq:
6095 case clang::AArch64::BI_InterlockedAdd64_acq:
6096 Ordering = llvm::AtomicOrdering::Acquire;
6097 break;
6098 case clang::AArch64::BI_InterlockedAdd_rel:
6099 case clang::AArch64::BI_InterlockedAdd64_rel:
6100 Ordering = llvm::AtomicOrdering::Release;
6101 break;
6102 case clang::AArch64::BI_InterlockedAdd_nf:
6103 case clang::AArch64::BI_InterlockedAdd64_nf:
6104 Ordering = llvm::AtomicOrdering::Monotonic;
6105 break;
6106 default:
6107 llvm_unreachable("missing builtin ID in switch!");
6108 }
6109 AtomicRMWInst *RMWI =
6110 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Addr: DestAddr, Val, Ordering);
6111 return Builder.CreateAdd(LHS: RMWI, RHS: Val);
6112 }
6113 }
6114
6115 llvm::FixedVectorType *VTy = GetNeonType(CGF: this, TypeFlags: Type);
6116 llvm::Type *Ty = VTy;
6117 if (!Ty)
6118 return nullptr;
6119
6120 bool ExtractLow = false;
6121 bool ExtendLaneArg = false;
6122 switch (BuiltinID) {
6123 default: return nullptr;
6124 case NEON::BI__builtin_neon_vbsl_v:
6125 case NEON::BI__builtin_neon_vbslq_v: {
6126 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
6127 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: BitTy, Name: "vbsl");
6128 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: BitTy, Name: "vbsl");
6129 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: BitTy, Name: "vbsl");
6130
6131 Ops[1] = Builder.CreateAnd(LHS: Ops[0], RHS: Ops[1], Name: "vbsl");
6132 Ops[2] = Builder.CreateAnd(LHS: Builder.CreateNot(V: Ops[0]), RHS: Ops[2], Name: "vbsl");
6133 Ops[0] = Builder.CreateOr(LHS: Ops[1], RHS: Ops[2], Name: "vbsl");
6134 return Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6135 }
6136 case NEON::BI__builtin_neon_vfma_lane_v:
6137 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
6138 // The ARM builtins (and instructions) have the addend as the first
6139 // operand, but the 'fma' intrinsics have it last. Swap it around here.
6140 Value *Addend = Ops[0];
6141 Value *Multiplicand = Ops[1];
6142 Value *LaneSource = Ops[2];
6143 Ops[0] = Multiplicand;
6144 Ops[1] = LaneSource;
6145 Ops[2] = Addend;
6146
6147 // Now adjust things to handle the lane access.
6148 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
6149 ? llvm::FixedVectorType::get(ElementType: VTy->getElementType(),
6150 NumElts: VTy->getNumElements() / 2)
6151 : VTy;
6152 llvm::Constant *cst = cast<Constant>(Val: Ops[3]);
6153 Value *SV = llvm::ConstantVector::getSplat(EC: VTy->getElementCount(), Elt: cst);
6154 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: SourceTy);
6155 Ops[1] = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[1], Mask: SV, Name: "lane");
6156
6157 Ops.pop_back();
6158 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
6159 : Intrinsic::fma;
6160 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "fmla");
6161 }
6162 case NEON::BI__builtin_neon_vfma_laneq_v: {
6163 auto *VTy = cast<llvm::FixedVectorType>(Val: Ty);
6164 // v1f64 fma should be mapped to Neon scalar f64 fma
6165 if (VTy && VTy->getElementType() == DoubleTy) {
6166 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
6167 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: DoubleTy);
6168 llvm::FixedVectorType *VTy =
6169 GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float64, false, true));
6170 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: VTy);
6171 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "extract");
6172 Value *Result;
6173 Result = emitCallMaybeConstrainedFPBuiltin(
6174 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma,
6175 Ty: DoubleTy, Args: {Ops[1], Ops[2], Ops[0]});
6176 return Builder.CreateBitCast(V: Result, DestTy: Ty);
6177 }
6178 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6179 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6180
6181 auto *STy = llvm::FixedVectorType::get(ElementType: VTy->getElementType(),
6182 NumElts: VTy->getNumElements() * 2);
6183 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: STy);
6184 Value *SV = llvm::ConstantVector::getSplat(EC: VTy->getElementCount(),
6185 Elt: cast<ConstantInt>(Val: Ops[3]));
6186 Ops[2] = Builder.CreateShuffleVector(V1: Ops[2], V2: Ops[2], Mask: SV, Name: "lane");
6187
6188 return emitCallMaybeConstrainedFPBuiltin(
6189 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
6190 Args: {Ops[2], Ops[1], Ops[0]});
6191 }
6192 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
6193 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6194 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6195
6196 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6197 Ops[2] = EmitNeonSplat(V: Ops[2], C: cast<ConstantInt>(Val: Ops[3]));
6198 return emitCallMaybeConstrainedFPBuiltin(
6199 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
6200 Args: {Ops[2], Ops[1], Ops[0]});
6201 }
6202 case NEON::BI__builtin_neon_vfmah_lane_f16:
6203 case NEON::BI__builtin_neon_vfmas_lane_f32:
6204 case NEON::BI__builtin_neon_vfmah_laneq_f16:
6205 case NEON::BI__builtin_neon_vfmas_laneq_f32:
6206 case NEON::BI__builtin_neon_vfmad_lane_f64:
6207 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
6208 llvm::Type *Ty = ConvertType(T: E->getCallReturnType(Ctx: getContext()));
6209 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "extract");
6210 return emitCallMaybeConstrainedFPBuiltin(
6211 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
6212 Args: {Ops[1], Ops[2], Ops[0]});
6213 }
6214 case NEON::BI__builtin_neon_vmull_v:
6215 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6216 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
6217 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
6218 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmull");
6219 case NEON::BI__builtin_neon_vmax_v:
6220 case NEON::BI__builtin_neon_vmaxq_v:
6221 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6222 Int = usgn ? Intrinsic::umax : Intrinsic::smax;
6223 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
6224 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmax");
6225 case NEON::BI__builtin_neon_vmaxh_f16: {
6226 Int = Intrinsic::aarch64_neon_fmax;
6227 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmax");
6228 }
6229 case NEON::BI__builtin_neon_vmin_v:
6230 case NEON::BI__builtin_neon_vminq_v:
6231 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6232 Int = usgn ? Intrinsic::umin : Intrinsic::smin;
6233 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
6234 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmin");
6235 case NEON::BI__builtin_neon_vminh_f16: {
6236 Int = Intrinsic::aarch64_neon_fmin;
6237 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmin");
6238 }
6239 case NEON::BI__builtin_neon_vabd_v:
6240 case NEON::BI__builtin_neon_vabdq_v:
6241 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6242 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
6243 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
6244 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vabd");
6245 case NEON::BI__builtin_neon_vpadal_v:
6246 case NEON::BI__builtin_neon_vpadalq_v: {
6247 unsigned ArgElts = VTy->getNumElements();
6248 llvm::IntegerType *EltTy = cast<IntegerType>(Val: VTy->getElementType());
6249 unsigned BitWidth = EltTy->getBitWidth();
6250 auto *ArgTy = llvm::FixedVectorType::get(
6251 ElementType: llvm::IntegerType::get(C&: getLLVMContext(), NumBits: BitWidth / 2), NumElts: 2 * ArgElts);
6252 llvm::Type* Tys[2] = { VTy, ArgTy };
6253 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
6254 SmallVector<llvm::Value*, 1> TmpOps;
6255 TmpOps.push_back(Elt: Ops[1]);
6256 Function *F = CGM.getIntrinsic(IID: Int, Tys);
6257 llvm::Value *tmp = EmitNeonCall(F, Ops&: TmpOps, name: "vpadal");
6258 llvm::Value *addend = Builder.CreateBitCast(V: Ops[0], DestTy: tmp->getType());
6259 return Builder.CreateAdd(LHS: tmp, RHS: addend);
6260 }
6261 case NEON::BI__builtin_neon_vpmin_v:
6262 case NEON::BI__builtin_neon_vpminq_v:
6263 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6264 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
6265 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
6266 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpmin");
6267 case NEON::BI__builtin_neon_vpmax_v:
6268 case NEON::BI__builtin_neon_vpmaxq_v:
6269 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6270 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
6271 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
6272 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpmax");
6273 case NEON::BI__builtin_neon_vminnm_v:
6274 case NEON::BI__builtin_neon_vminnmq_v:
6275 Int = Intrinsic::aarch64_neon_fminnm;
6276 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vminnm");
6277 case NEON::BI__builtin_neon_vminnmh_f16:
6278 Int = Intrinsic::aarch64_neon_fminnm;
6279 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vminnm");
6280 case NEON::BI__builtin_neon_vmaxnm_v:
6281 case NEON::BI__builtin_neon_vmaxnmq_v:
6282 Int = Intrinsic::aarch64_neon_fmaxnm;
6283 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmaxnm");
6284 case NEON::BI__builtin_neon_vmaxnmh_f16:
6285 Int = Intrinsic::aarch64_neon_fmaxnm;
6286 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmaxnm");
6287 case NEON::BI__builtin_neon_vrecpss_f32: {
6288 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_frecps, Tys: FloatTy),
6289 Ops, name: "vrecps");
6290 }
6291 case NEON::BI__builtin_neon_vrecpsd_f64:
6292 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_frecps, Tys: DoubleTy),
6293 Ops, name: "vrecps");
6294 case NEON::BI__builtin_neon_vrecpsh_f16:
6295 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_frecps, Tys: HalfTy),
6296 Ops, name: "vrecps");
6297 case NEON::BI__builtin_neon_vqshrun_n_v:
6298 Int = Intrinsic::aarch64_neon_sqshrun;
6299 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshrun_n");
6300 case NEON::BI__builtin_neon_vqrshrun_n_v:
6301 Int = Intrinsic::aarch64_neon_sqrshrun;
6302 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqrshrun_n");
6303 case NEON::BI__builtin_neon_vqshrn_n_v:
6304 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
6305 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshrn_n");
6306 case NEON::BI__builtin_neon_vrshrn_n_v:
6307 Int = Intrinsic::aarch64_neon_rshrn;
6308 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrshrn_n");
6309 case NEON::BI__builtin_neon_vqrshrn_n_v:
6310 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
6311 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqrshrn_n");
6312 case NEON::BI__builtin_neon_vrndah_f16: {
6313 Int = Builder.getIsFPConstrained()
6314 ? Intrinsic::experimental_constrained_round
6315 : Intrinsic::round;
6316 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrnda");
6317 }
6318 case NEON::BI__builtin_neon_vrnda_v:
6319 case NEON::BI__builtin_neon_vrndaq_v: {
6320 Int = Builder.getIsFPConstrained()
6321 ? Intrinsic::experimental_constrained_round
6322 : Intrinsic::round;
6323 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrnda");
6324 }
6325 case NEON::BI__builtin_neon_vrndih_f16: {
6326 Int = Builder.getIsFPConstrained()
6327 ? Intrinsic::experimental_constrained_nearbyint
6328 : Intrinsic::nearbyint;
6329 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndi");
6330 }
6331 case NEON::BI__builtin_neon_vrndmh_f16: {
6332 Int = Builder.getIsFPConstrained()
6333 ? Intrinsic::experimental_constrained_floor
6334 : Intrinsic::floor;
6335 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndm");
6336 }
6337 case NEON::BI__builtin_neon_vrndm_v:
6338 case NEON::BI__builtin_neon_vrndmq_v: {
6339 Int = Builder.getIsFPConstrained()
6340 ? Intrinsic::experimental_constrained_floor
6341 : Intrinsic::floor;
6342 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndm");
6343 }
6344 case NEON::BI__builtin_neon_vrndnh_f16: {
6345 Int = Builder.getIsFPConstrained()
6346 ? Intrinsic::experimental_constrained_roundeven
6347 : Intrinsic::roundeven;
6348 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndn");
6349 }
6350 case NEON::BI__builtin_neon_vrndn_v:
6351 case NEON::BI__builtin_neon_vrndnq_v: {
6352 Int = Builder.getIsFPConstrained()
6353 ? Intrinsic::experimental_constrained_roundeven
6354 : Intrinsic::roundeven;
6355 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndn");
6356 }
6357 case NEON::BI__builtin_neon_vrndns_f32: {
6358 Int = Builder.getIsFPConstrained()
6359 ? Intrinsic::experimental_constrained_roundeven
6360 : Intrinsic::roundeven;
6361 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: FloatTy), Ops, name: "vrndn");
6362 }
6363 case NEON::BI__builtin_neon_vrndph_f16: {
6364 Int = Builder.getIsFPConstrained()
6365 ? Intrinsic::experimental_constrained_ceil
6366 : Intrinsic::ceil;
6367 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndp");
6368 }
6369 case NEON::BI__builtin_neon_vrndp_v:
6370 case NEON::BI__builtin_neon_vrndpq_v: {
6371 Int = Builder.getIsFPConstrained()
6372 ? Intrinsic::experimental_constrained_ceil
6373 : Intrinsic::ceil;
6374 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndp");
6375 }
6376 case NEON::BI__builtin_neon_vrndxh_f16: {
6377 Int = Builder.getIsFPConstrained()
6378 ? Intrinsic::experimental_constrained_rint
6379 : Intrinsic::rint;
6380 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndx");
6381 }
6382 case NEON::BI__builtin_neon_vrndx_v:
6383 case NEON::BI__builtin_neon_vrndxq_v: {
6384 Int = Builder.getIsFPConstrained()
6385 ? Intrinsic::experimental_constrained_rint
6386 : Intrinsic::rint;
6387 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndx");
6388 }
6389 case NEON::BI__builtin_neon_vrndh_f16: {
6390 Int = Builder.getIsFPConstrained()
6391 ? Intrinsic::experimental_constrained_trunc
6392 : Intrinsic::trunc;
6393 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndz");
6394 }
6395 case NEON::BI__builtin_neon_vrnd_v:
6396 case NEON::BI__builtin_neon_vrndq_v: {
6397 Int = Builder.getIsFPConstrained()
6398 ? Intrinsic::experimental_constrained_trunc
6399 : Intrinsic::trunc;
6400 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndz");
6401 }
6402 case NEON::BI__builtin_neon_vcvt_f64_v:
6403 case NEON::BI__builtin_neon_vcvtq_f64_v:
6404 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6405 Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
6406 return usgn ? Builder.CreateUIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt")
6407 : Builder.CreateSIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt");
6408 case NEON::BI__builtin_neon_vcvt_f64_f32: {
6409 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&
6410 "unexpected vcvt_f64_f32 builtin");
6411 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
6412 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: GetNeonType(CGF: this, TypeFlags: SrcFlag));
6413
6414 return Builder.CreateFPExt(V: Ops[0], DestTy: Ty, Name: "vcvt");
6415 }
6416 case NEON::BI__builtin_neon_vcvt_f32_f64: {
6417 assert(Type.getEltType() == NeonTypeFlags::Float32 &&
6418 "unexpected vcvt_f32_f64 builtin");
6419 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
6420 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: GetNeonType(CGF: this, TypeFlags: SrcFlag));
6421
6422 return Builder.CreateFPTrunc(V: Ops[0], DestTy: Ty, Name: "vcvt");
6423 }
6424 case NEON::BI__builtin_neon_vcvta_s16_f16:
6425 case NEON::BI__builtin_neon_vcvta_u16_f16:
6426 case NEON::BI__builtin_neon_vcvta_s32_v:
6427 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
6428 case NEON::BI__builtin_neon_vcvtaq_s32_v:
6429 case NEON::BI__builtin_neon_vcvta_u32_v:
6430 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
6431 case NEON::BI__builtin_neon_vcvtaq_u32_v:
6432 case NEON::BI__builtin_neon_vcvta_s64_v:
6433 case NEON::BI__builtin_neon_vcvtaq_s64_v:
6434 case NEON::BI__builtin_neon_vcvta_u64_v:
6435 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
6436 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
6437 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6438 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvta");
6439 }
6440 case NEON::BI__builtin_neon_vcvtm_s16_f16:
6441 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
6442 case NEON::BI__builtin_neon_vcvtm_u16_f16:
6443 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
6444 case NEON::BI__builtin_neon_vcvtm_s32_v:
6445 case NEON::BI__builtin_neon_vcvtmq_s32_v:
6446 case NEON::BI__builtin_neon_vcvtm_u32_v:
6447 case NEON::BI__builtin_neon_vcvtmq_u32_v:
6448 case NEON::BI__builtin_neon_vcvtm_s64_v:
6449 case NEON::BI__builtin_neon_vcvtmq_s64_v:
6450 case NEON::BI__builtin_neon_vcvtm_u64_v:
6451 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
6452 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
6453 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6454 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtm");
6455 }
6456 case NEON::BI__builtin_neon_vcvtn_s16_f16:
6457 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
6458 case NEON::BI__builtin_neon_vcvtn_u16_f16:
6459 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
6460 case NEON::BI__builtin_neon_vcvtn_s32_v:
6461 case NEON::BI__builtin_neon_vcvtnq_s32_v:
6462 case NEON::BI__builtin_neon_vcvtn_u32_v:
6463 case NEON::BI__builtin_neon_vcvtnq_u32_v:
6464 case NEON::BI__builtin_neon_vcvtn_s64_v:
6465 case NEON::BI__builtin_neon_vcvtnq_s64_v:
6466 case NEON::BI__builtin_neon_vcvtn_u64_v:
6467 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
6468 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
6469 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6470 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtn");
6471 }
6472 case NEON::BI__builtin_neon_vcvtp_s16_f16:
6473 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
6474 case NEON::BI__builtin_neon_vcvtp_u16_f16:
6475 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
6476 case NEON::BI__builtin_neon_vcvtp_s32_v:
6477 case NEON::BI__builtin_neon_vcvtpq_s32_v:
6478 case NEON::BI__builtin_neon_vcvtp_u32_v:
6479 case NEON::BI__builtin_neon_vcvtpq_u32_v:
6480 case NEON::BI__builtin_neon_vcvtp_s64_v:
6481 case NEON::BI__builtin_neon_vcvtpq_s64_v:
6482 case NEON::BI__builtin_neon_vcvtp_u64_v:
6483 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
6484 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
6485 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6486 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtp");
6487 }
6488 case NEON::BI__builtin_neon_vmulx_v:
6489 case NEON::BI__builtin_neon_vmulxq_v: {
6490 Int = Intrinsic::aarch64_neon_fmulx;
6491 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmulx");
6492 }
6493 case NEON::BI__builtin_neon_vmulxh_lane_f16:
6494 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
6495 // vmulx_lane should be mapped to Neon scalar mulx after
6496 // extracting the scalar element
6497 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2], Name: "extract");
6498 Ops.pop_back();
6499 Int = Intrinsic::aarch64_neon_fmulx;
6500 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmulx");
6501 }
6502 case NEON::BI__builtin_neon_vmul_lane_v:
6503 case NEON::BI__builtin_neon_vmul_laneq_v: {
6504 // v1f64 vmul_lane should be mapped to Neon scalar mul lane
6505 bool Quad = false;
6506 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
6507 Quad = true;
6508 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
6509 llvm::FixedVectorType *VTy =
6510 GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
6511 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: VTy);
6512 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2], Name: "extract");
6513 Value *Result = Builder.CreateFMul(L: Ops[0], R: Ops[1]);
6514 return Builder.CreateBitCast(V: Result, DestTy: Ty);
6515 }
6516 case NEON::BI__builtin_neon_vpmaxnm_v:
6517 case NEON::BI__builtin_neon_vpmaxnmq_v: {
6518 Int = Intrinsic::aarch64_neon_fmaxnmp;
6519 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpmaxnm");
6520 }
6521 case NEON::BI__builtin_neon_vpminnm_v:
6522 case NEON::BI__builtin_neon_vpminnmq_v: {
6523 Int = Intrinsic::aarch64_neon_fminnmp;
6524 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpminnm");
6525 }
6526 case NEON::BI__builtin_neon_vsqrth_f16: {
6527 Int = Builder.getIsFPConstrained()
6528 ? Intrinsic::experimental_constrained_sqrt
6529 : Intrinsic::sqrt;
6530 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vsqrt");
6531 }
6532 case NEON::BI__builtin_neon_vsqrt_v:
6533 case NEON::BI__builtin_neon_vsqrtq_v: {
6534 Int = Builder.getIsFPConstrained()
6535 ? Intrinsic::experimental_constrained_sqrt
6536 : Intrinsic::sqrt;
6537 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6538 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vsqrt");
6539 }
6540 case NEON::BI__builtin_neon_vrbit_v:
6541 case NEON::BI__builtin_neon_vrbitq_v: {
6542 Int = Intrinsic::bitreverse;
6543 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrbit");
6544 }
6545 case NEON::BI__builtin_neon_vmaxv_f16: {
6546 Int = Intrinsic::aarch64_neon_fmaxv;
6547 Ty = HalfTy;
6548 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6549 llvm::Type *Tys[2] = {Ty, VTy};
6550 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxv");
6551 }
6552 case NEON::BI__builtin_neon_vmaxvq_f16: {
6553 Int = Intrinsic::aarch64_neon_fmaxv;
6554 Ty = HalfTy;
6555 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6556 llvm::Type *Tys[2] = {Ty, VTy};
6557 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxv");
6558 }
6559 case NEON::BI__builtin_neon_vminv_f16: {
6560 Int = Intrinsic::aarch64_neon_fminv;
6561 Ty = HalfTy;
6562 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6563 llvm::Type *Tys[2] = {Ty, VTy};
6564 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminv");
6565 }
6566 case NEON::BI__builtin_neon_vminvq_f16: {
6567 Int = Intrinsic::aarch64_neon_fminv;
6568 Ty = HalfTy;
6569 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6570 llvm::Type *Tys[2] = {Ty, VTy};
6571 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminv");
6572 }
6573 case NEON::BI__builtin_neon_vmaxnmv_f16: {
6574 Int = Intrinsic::aarch64_neon_fmaxnmv;
6575 Ty = HalfTy;
6576 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6577 llvm::Type *Tys[2] = {Ty, VTy};
6578 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxnmv");
6579 }
6580 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
6581 Int = Intrinsic::aarch64_neon_fmaxnmv;
6582 Ty = HalfTy;
6583 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6584 llvm::Type *Tys[2] = {Ty, VTy};
6585 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxnmv");
6586 }
6587 case NEON::BI__builtin_neon_vminnmv_f16: {
6588 Int = Intrinsic::aarch64_neon_fminnmv;
6589 Ty = HalfTy;
6590 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6591 llvm::Type *Tys[2] = {Ty, VTy};
6592 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminnmv");
6593 }
6594 case NEON::BI__builtin_neon_vminnmvq_f16: {
6595 Int = Intrinsic::aarch64_neon_fminnmv;
6596 Ty = HalfTy;
6597 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6598 llvm::Type *Tys[2] = {Ty, VTy};
6599 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminnmv");
6600 }
6601 case NEON::BI__builtin_neon_vmul_n_f64: {
6602 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
6603 Value *RHS = Builder.CreateBitCast(V: Ops[1], DestTy: DoubleTy);
6604 return Builder.CreateFMul(L: Ops[0], R: RHS);
6605 }
6606 case NEON::BI__builtin_neon_vaddlv_u8:
6607 case NEON::BI__builtin_neon_vaddlvq_u8:
6608 case NEON::BI__builtin_neon_vaddlv_u16:
6609 case NEON::BI__builtin_neon_vaddlvq_u16: {
6610 Int = Intrinsic::aarch64_neon_uaddlv;
6611 Ty = Int32Ty;
6612 VTy = cast<llvm::FixedVectorType>(Val: Ops[0]->getType());
6613 llvm::Type *Tys[2] = {Ty, VTy};
6614 Value *Result = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vaddlv");
6615 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6616 return Builder.CreateTrunc(V: Result, DestTy: Int16Ty);
6617 return Result;
6618 }
6619 case NEON::BI__builtin_neon_vaddlv_s8:
6620 case NEON::BI__builtin_neon_vaddlvq_s8:
6621 case NEON::BI__builtin_neon_vaddlv_s16:
6622 case NEON::BI__builtin_neon_vaddlvq_s16: {
6623 Int = Intrinsic::aarch64_neon_saddlv;
6624 Ty = Int32Ty;
6625 VTy = cast<llvm::FixedVectorType>(Val: Ops[0]->getType());
6626 llvm::Type *Tys[2] = {Ty, VTy};
6627 Value *Result = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vaddlv");
6628 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6629 return Builder.CreateTrunc(V: Result, DestTy: Int16Ty);
6630 return Result;
6631 }
6632 case NEON::BI__builtin_neon_vsri_n_v:
6633 case NEON::BI__builtin_neon_vsriq_n_v: {
6634 Int = Intrinsic::aarch64_neon_vsri;
6635 llvm::Function *Intrin = CGM.getIntrinsic(IID: Int, Tys: Ty);
6636 return EmitNeonCall(F: Intrin, Ops, name: "vsri_n");
6637 }
6638 case NEON::BI__builtin_neon_vsli_n_v:
6639 case NEON::BI__builtin_neon_vsliq_n_v: {
6640 Int = Intrinsic::aarch64_neon_vsli;
6641 llvm::Function *Intrin = CGM.getIntrinsic(IID: Int, Tys: Ty);
6642 return EmitNeonCall(F: Intrin, Ops, name: "vsli_n");
6643 }
6644 case NEON::BI__builtin_neon_vsra_n_v:
6645 case NEON::BI__builtin_neon_vsraq_n_v:
6646 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6647 Ops[1] = EmitNeonRShiftImm(Vec: Ops[1], Shift: Ops[2], Ty, usgn, name: "vsra_n");
6648 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
6649 case NEON::BI__builtin_neon_vrsra_n_v:
6650 case NEON::BI__builtin_neon_vrsraq_n_v: {
6651 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
6652 SmallVector<llvm::Value*,2> TmpOps;
6653 TmpOps.push_back(Elt: Ops[1]);
6654 TmpOps.push_back(Elt: Ops[2]);
6655 Function* F = CGM.getIntrinsic(IID: Int, Tys: Ty);
6656 llvm::Value *tmp = EmitNeonCall(F, Ops&: TmpOps, name: "vrshr_n", shift: 1, rightshift: true);
6657 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: VTy);
6658 return Builder.CreateAdd(LHS: Ops[0], RHS: tmp);
6659 }
6660 case NEON::BI__builtin_neon_vld1_v:
6661 case NEON::BI__builtin_neon_vld1q_v: {
6662 return Builder.CreateAlignedLoad(Ty: VTy, Addr: Ops[0], Align: PtrOp0.getAlignment());
6663 }
6664 case NEON::BI__builtin_neon_vst1_v:
6665 case NEON::BI__builtin_neon_vst1q_v:
6666 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: VTy);
6667 return Builder.CreateAlignedStore(Val: Ops[1], Addr: Ops[0], Align: PtrOp0.getAlignment());
6668 case NEON::BI__builtin_neon_vld1_lane_v:
6669 case NEON::BI__builtin_neon_vld1q_lane_v: {
6670 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6671 Ops[0] = Builder.CreateAlignedLoad(Ty: VTy->getElementType(), Addr: Ops[0],
6672 Align: PtrOp0.getAlignment());
6673 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vld1_lane");
6674 }
6675 case NEON::BI__builtin_neon_vldap1_lane_s64:
6676 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
6677 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6678 llvm::LoadInst *LI = Builder.CreateAlignedLoad(
6679 Ty: VTy->getElementType(), Addr: Ops[0], Align: PtrOp0.getAlignment());
6680 LI->setAtomic(Ordering: llvm::AtomicOrdering::Acquire);
6681 Ops[0] = LI;
6682 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vldap1_lane");
6683 }
6684 case NEON::BI__builtin_neon_vld1_dup_v:
6685 case NEON::BI__builtin_neon_vld1q_dup_v: {
6686 Value *V = PoisonValue::get(T: Ty);
6687 Ops[0] = Builder.CreateAlignedLoad(Ty: VTy->getElementType(), Addr: Ops[0],
6688 Align: PtrOp0.getAlignment());
6689 llvm::Constant *CI = ConstantInt::get(Ty: Int32Ty, V: 0);
6690 Ops[0] = Builder.CreateInsertElement(Vec: V, NewElt: Ops[0], Idx: CI);
6691 return EmitNeonSplat(V: Ops[0], C: CI);
6692 }
6693 case NEON::BI__builtin_neon_vst1_lane_v:
6694 case NEON::BI__builtin_neon_vst1q_lane_v:
6695 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6696 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2]);
6697 return Builder.CreateAlignedStore(Val: Ops[1], Addr: Ops[0], Align: PtrOp0.getAlignment());
6698 case NEON::BI__builtin_neon_vstl1_lane_s64:
6699 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
6700 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6701 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2]);
6702 llvm::StoreInst *SI =
6703 Builder.CreateAlignedStore(Val: Ops[1], Addr: Ops[0], Align: PtrOp0.getAlignment());
6704 SI->setAtomic(Ordering: llvm::AtomicOrdering::Release);
6705 return SI;
6706 }
6707 case NEON::BI__builtin_neon_vld2_v:
6708 case NEON::BI__builtin_neon_vld2q_v: {
6709 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6710 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld2, Tys);
6711 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld2");
6712 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6713 }
6714 case NEON::BI__builtin_neon_vld3_v:
6715 case NEON::BI__builtin_neon_vld3q_v: {
6716 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6717 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld3, Tys);
6718 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld3");
6719 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6720 }
6721 case NEON::BI__builtin_neon_vld4_v:
6722 case NEON::BI__builtin_neon_vld4q_v: {
6723 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6724 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld4, Tys);
6725 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld4");
6726 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6727 }
6728 case NEON::BI__builtin_neon_vld2_dup_v:
6729 case NEON::BI__builtin_neon_vld2q_dup_v: {
6730 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6731 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld2r, Tys);
6732 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld2");
6733 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6734 }
6735 case NEON::BI__builtin_neon_vld3_dup_v:
6736 case NEON::BI__builtin_neon_vld3q_dup_v: {
6737 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6738 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld3r, Tys);
6739 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld3");
6740 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6741 }
6742 case NEON::BI__builtin_neon_vld4_dup_v:
6743 case NEON::BI__builtin_neon_vld4q_dup_v: {
6744 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6745 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld4r, Tys);
6746 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld4");
6747 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6748 }
6749 case NEON::BI__builtin_neon_vld2_lane_v:
6750 case NEON::BI__builtin_neon_vld2q_lane_v: {
6751 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6752 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld2lane, Tys);
6753 std::rotate(first: Ops.begin() + 1, middle: Ops.begin() + 2, last: Ops.end());
6754 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6755 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6756 Ops[3] = Builder.CreateZExt(V: Ops[3], DestTy: Int64Ty);
6757 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: "vld2_lane");
6758 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6759 }
6760 case NEON::BI__builtin_neon_vld3_lane_v:
6761 case NEON::BI__builtin_neon_vld3q_lane_v: {
6762 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6763 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld3lane, Tys);
6764 std::rotate(first: Ops.begin() + 1, middle: Ops.begin() + 2, last: Ops.end());
6765 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6766 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6767 Ops[3] = Builder.CreateBitCast(V: Ops[3], DestTy: Ty);
6768 Ops[4] = Builder.CreateZExt(V: Ops[4], DestTy: Int64Ty);
6769 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: "vld3_lane");
6770 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6771 }
6772 case NEON::BI__builtin_neon_vld4_lane_v:
6773 case NEON::BI__builtin_neon_vld4q_lane_v: {
6774 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6775 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld4lane, Tys);
6776 std::rotate(first: Ops.begin() + 1, middle: Ops.begin() + 2, last: Ops.end());
6777 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6778 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6779 Ops[3] = Builder.CreateBitCast(V: Ops[3], DestTy: Ty);
6780 Ops[4] = Builder.CreateBitCast(V: Ops[4], DestTy: Ty);
6781 Ops[5] = Builder.CreateZExt(V: Ops[5], DestTy: Int64Ty);
6782 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: "vld4_lane");
6783 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6784 }
6785 case NEON::BI__builtin_neon_vst2_v:
6786 case NEON::BI__builtin_neon_vst2q_v: {
6787 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6788 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
6789 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st2, Tys),
6790 Ops, name: "");
6791 }
6792 case NEON::BI__builtin_neon_vst2_lane_v:
6793 case NEON::BI__builtin_neon_vst2q_lane_v: {
6794 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6795 Ops[2] = Builder.CreateZExt(V: Ops[2], DestTy: Int64Ty);
6796 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6797 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st2lane, Tys),
6798 Ops, name: "");
6799 }
6800 case NEON::BI__builtin_neon_vst3_v:
6801 case NEON::BI__builtin_neon_vst3q_v: {
6802 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6803 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6804 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st3, Tys),
6805 Ops, name: "");
6806 }
6807 case NEON::BI__builtin_neon_vst3_lane_v:
6808 case NEON::BI__builtin_neon_vst3q_lane_v: {
6809 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6810 Ops[3] = Builder.CreateZExt(V: Ops[3], DestTy: Int64Ty);
6811 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6812 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st3lane, Tys),
6813 Ops, name: "");
6814 }
6815 case NEON::BI__builtin_neon_vst4_v:
6816 case NEON::BI__builtin_neon_vst4q_v: {
6817 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6818 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6819 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st4, Tys),
6820 Ops, name: "");
6821 }
6822 case NEON::BI__builtin_neon_vst4_lane_v:
6823 case NEON::BI__builtin_neon_vst4q_lane_v: {
6824 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6825 Ops[4] = Builder.CreateZExt(V: Ops[4], DestTy: Int64Ty);
6826 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
6827 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st4lane, Tys),
6828 Ops, name: "");
6829 }
6830 case NEON::BI__builtin_neon_vtrn_v:
6831 case NEON::BI__builtin_neon_vtrnq_v: {
6832 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6833 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6834 Value *SV = nullptr;
6835
6836 for (unsigned vi = 0; vi != 2; ++vi) {
6837 SmallVector<int, 16> Indices;
6838 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6839 Indices.push_back(Elt: i+vi);
6840 Indices.push_back(Elt: i+e+vi);
6841 }
6842 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
6843 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vtrn");
6844 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
6845 }
6846 return SV;
6847 }
6848 case NEON::BI__builtin_neon_vuzp_v:
6849 case NEON::BI__builtin_neon_vuzpq_v: {
6850 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6851 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6852 Value *SV = nullptr;
6853
6854 for (unsigned vi = 0; vi != 2; ++vi) {
6855 SmallVector<int, 16> Indices;
6856 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
6857 Indices.push_back(Elt: 2*i+vi);
6858
6859 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
6860 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vuzp");
6861 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
6862 }
6863 return SV;
6864 }
6865 case NEON::BI__builtin_neon_vzip_v:
6866 case NEON::BI__builtin_neon_vzipq_v: {
6867 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6868 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6869 Value *SV = nullptr;
6870
6871 for (unsigned vi = 0; vi != 2; ++vi) {
6872 SmallVector<int, 16> Indices;
6873 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6874 Indices.push_back(Elt: (i + vi*e) >> 1);
6875 Indices.push_back(Elt: ((i + vi*e) >> 1)+e);
6876 }
6877 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
6878 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vzip");
6879 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
6880 }
6881 return SV;
6882 }
6883 case NEON::BI__builtin_neon_vqtbl1q_v: {
6884 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl1, Tys: Ty),
6885 Ops, name: "vtbl1");
6886 }
6887 case NEON::BI__builtin_neon_vqtbl2q_v: {
6888 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl2, Tys: Ty),
6889 Ops, name: "vtbl2");
6890 }
6891 case NEON::BI__builtin_neon_vqtbl3q_v: {
6892 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl3, Tys: Ty),
6893 Ops, name: "vtbl3");
6894 }
6895 case NEON::BI__builtin_neon_vqtbl4q_v: {
6896 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl4, Tys: Ty),
6897 Ops, name: "vtbl4");
6898 }
6899 case NEON::BI__builtin_neon_vqtbx1q_v: {
6900 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx1, Tys: Ty),
6901 Ops, name: "vtbx1");
6902 }
6903 case NEON::BI__builtin_neon_vqtbx2q_v: {
6904 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx2, Tys: Ty),
6905 Ops, name: "vtbx2");
6906 }
6907 case NEON::BI__builtin_neon_vqtbx3q_v: {
6908 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx3, Tys: Ty),
6909 Ops, name: "vtbx3");
6910 }
6911 case NEON::BI__builtin_neon_vqtbx4q_v: {
6912 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx4, Tys: Ty),
6913 Ops, name: "vtbx4");
6914 }
6915 case NEON::BI__builtin_neon_vsqadd_v:
6916 case NEON::BI__builtin_neon_vsqaddq_v: {
6917 Int = Intrinsic::aarch64_neon_usqadd;
6918 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vsqadd");
6919 }
6920 case NEON::BI__builtin_neon_vuqadd_v:
6921 case NEON::BI__builtin_neon_vuqaddq_v: {
6922 Int = Intrinsic::aarch64_neon_suqadd;
6923 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vuqadd");
6924 }
6925
6926 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
6927 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
6928 case NEON::BI__builtin_neon_vluti2_laneq_f16:
6929 case NEON::BI__builtin_neon_vluti2_laneq_p16:
6930 case NEON::BI__builtin_neon_vluti2_laneq_p8:
6931 case NEON::BI__builtin_neon_vluti2_laneq_s16:
6932 case NEON::BI__builtin_neon_vluti2_laneq_s8:
6933 case NEON::BI__builtin_neon_vluti2_laneq_u16:
6934 case NEON::BI__builtin_neon_vluti2_laneq_u8: {
6935 Int = Intrinsic::aarch64_neon_vluti2_laneq;
6936 llvm::Type *Tys[2];
6937 Tys[0] = Ty;
6938 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
6939 /*isQuad*/ false));
6940 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_laneq");
6941 }
6942 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
6943 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
6944 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
6945 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
6946 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
6947 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
6948 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
6949 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
6950 case NEON::BI__builtin_neon_vluti2q_laneq_u8: {
6951 Int = Intrinsic::aarch64_neon_vluti2_laneq;
6952 llvm::Type *Tys[2];
6953 Tys[0] = Ty;
6954 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
6955 /*isQuad*/ true));
6956 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_laneq");
6957 }
6958 case NEON::BI__builtin_neon_vluti2_lane_mf8:
6959 case NEON::BI__builtin_neon_vluti2_lane_bf16:
6960 case NEON::BI__builtin_neon_vluti2_lane_f16:
6961 case NEON::BI__builtin_neon_vluti2_lane_p16:
6962 case NEON::BI__builtin_neon_vluti2_lane_p8:
6963 case NEON::BI__builtin_neon_vluti2_lane_s16:
6964 case NEON::BI__builtin_neon_vluti2_lane_s8:
6965 case NEON::BI__builtin_neon_vluti2_lane_u16:
6966 case NEON::BI__builtin_neon_vluti2_lane_u8: {
6967 Int = Intrinsic::aarch64_neon_vluti2_lane;
6968 llvm::Type *Tys[2];
6969 Tys[0] = Ty;
6970 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
6971 /*isQuad*/ false));
6972 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_lane");
6973 }
6974 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
6975 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
6976 case NEON::BI__builtin_neon_vluti2q_lane_f16:
6977 case NEON::BI__builtin_neon_vluti2q_lane_p16:
6978 case NEON::BI__builtin_neon_vluti2q_lane_p8:
6979 case NEON::BI__builtin_neon_vluti2q_lane_s16:
6980 case NEON::BI__builtin_neon_vluti2q_lane_s8:
6981 case NEON::BI__builtin_neon_vluti2q_lane_u16:
6982 case NEON::BI__builtin_neon_vluti2q_lane_u8: {
6983 Int = Intrinsic::aarch64_neon_vluti2_lane;
6984 llvm::Type *Tys[2];
6985 Tys[0] = Ty;
6986 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
6987 /*isQuad*/ true));
6988 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_lane");
6989 }
6990 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
6991 case NEON::BI__builtin_neon_vluti4q_lane_p8:
6992 case NEON::BI__builtin_neon_vluti4q_lane_s8:
6993 case NEON::BI__builtin_neon_vluti4q_lane_u8: {
6994 Int = Intrinsic::aarch64_neon_vluti4q_lane;
6995 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_lane");
6996 }
6997 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
6998 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
6999 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
7000 case NEON::BI__builtin_neon_vluti4q_laneq_u8: {
7001 Int = Intrinsic::aarch64_neon_vluti4q_laneq;
7002 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_laneq");
7003 }
7004 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
7005 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
7006 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
7007 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
7008 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: {
7009 Int = Intrinsic::aarch64_neon_vluti4q_lane_x2;
7010 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_lane_x2");
7011 }
7012 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
7013 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
7014 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
7015 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
7016 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: {
7017 Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
7018 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_laneq_x2");
7019 }
7020 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
7021 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fmmla,
7022 Tys: {llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8),
7023 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16)},
7024 Ops, E, name: "fmmla");
7025 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
7026 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fmmla,
7027 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4),
7028 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16)},
7029 Ops, E, name: "fmmla");
7030 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
7031 ExtractLow = true;
7032 [[fallthrough]];
7033 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
7034 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
7035 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl1,
7036 Ty0: llvm::FixedVectorType::get(ElementType: BFloatTy, NumElts: 8),
7037 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt1");
7038 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
7039 ExtractLow = true;
7040 [[fallthrough]];
7041 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
7042 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
7043 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl2,
7044 Ty0: llvm::FixedVectorType::get(ElementType: BFloatTy, NumElts: 8),
7045 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt2");
7046 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
7047 ExtractLow = true;
7048 [[fallthrough]];
7049 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
7050 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
7051 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl1,
7052 Ty0: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8),
7053 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt1");
7054 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
7055 ExtractLow = true;
7056 [[fallthrough]];
7057 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
7058 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
7059 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl2,
7060 Ty0: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8),
7061 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt2");
7062 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
7063 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn,
7064 Ty0: llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 8),
7065 Ty1: Ops[0]->getType(), Extract: false, Ops, E, name: "vfcvtn");
7066 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
7067 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn,
7068 Ty0: llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 8),
7069 Ty1: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4), Extract: false, Ops,
7070 E, name: "vfcvtn");
7071 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
7072 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn,
7073 Ty0: llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16),
7074 Ty1: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8), Extract: false, Ops,
7075 E, name: "vfcvtn");
7076 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: {
7077 llvm::Type *Ty = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16);
7078 Ops[0] = Builder.CreateInsertVector(DstType: Ty, SrcVec: PoisonValue::get(T: Ty), SubVec: Ops[0],
7079 Idx: uint64_t(0));
7080 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn2, Ty0: Ty,
7081 Ty1: Ops[1]->getType(), Extract: false, Ops, E, name: "vfcvtn2");
7082 }
7083
7084 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
7085 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
7086 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot2, ExtendLaneArg: false, RetTy: HalfTy,
7087 Ops, E, name: "fdot2");
7088 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
7089 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
7090 ExtendLaneArg = true;
7091 [[fallthrough]];
7092 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
7093 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
7094 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot2_lane,
7095 ExtendLaneArg, RetTy: HalfTy, Ops, E, name: "fdot2_lane");
7096 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
7097 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
7098 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot4, ExtendLaneArg: false,
7099 RetTy: FloatTy, Ops, E, name: "fdot4");
7100 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
7101 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
7102 ExtendLaneArg = true;
7103 [[fallthrough]];
7104 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
7105 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
7106 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot4_lane,
7107 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "fdot4_lane");
7108
7109 case NEON::BI__builtin_neon_vdot_f32_f16:
7110 case NEON::BI__builtin_neon_vdotq_f32_f16: {
7111 llvm::Type *InputTy =
7112 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
7113 llvm::Type *Tys[2] = {Ty, InputTy};
7114 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_fdot, Tys),
7115 Ops, name: "vdot");
7116 }
7117
7118 case NEON::BI__builtin_neon_vdot_lane_f32_f16:
7119 case NEON::BI__builtin_neon_vdot_laneq_f32_f16:
7120 case NEON::BI__builtin_neon_vdotq_lane_f32_f16:
7121 case NEON::BI__builtin_neon_vdotq_laneq_f32_f16: {
7122 llvm::FixedVectorType *InputTy =
7123 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
7124 llvm::FixedVectorType *LaneTy = llvm::FixedVectorType::get(
7125 ElementType: HalfTy, NumElts: Ops[2]->getType()->getPrimitiveSizeInBits() / 16);
7126 // Treat the lane argument as a splat and use non-lane version of the
7127 // intrinsic.
7128 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: LaneTy);
7129 Ops[2] = EmitNeonSplat(V: Ops[2], C: cast<ConstantInt>(Val: Ops[3]),
7130 Count: InputTy->getElementCount());
7131 llvm::Type *Tys[2] = {Ty, InputTy};
7132 Ops.pop_back();
7133 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_fdot, Tys),
7134 Ops, name: "vdot");
7135 }
7136
7137 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
7138 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalb,
7139 Tys: {llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8)}, Ops, E,
7140 name: "vmlal");
7141 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
7142 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalt,
7143 Tys: {llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8)}, Ops, E,
7144 name: "vmlal");
7145 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
7146 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlallbb,
7147 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7148 name: "vmlall");
7149 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
7150 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlallbt,
7151 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7152 name: "vmlall");
7153 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
7154 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalltb,
7155 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7156 name: "vmlall");
7157 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
7158 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalltt,
7159 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7160 name: "vmlall");
7161 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
7162 ExtendLaneArg = true;
7163 [[fallthrough]];
7164 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
7165 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalb_lane,
7166 ExtendLaneArg, RetTy: HalfTy, Ops, E, name: "vmlal_lane");
7167 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
7168 ExtendLaneArg = true;
7169 [[fallthrough]];
7170 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
7171 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalt_lane,
7172 ExtendLaneArg, RetTy: HalfTy, Ops, E, name: "vmlal_lane");
7173 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
7174 ExtendLaneArg = true;
7175 [[fallthrough]];
7176 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
7177 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlallbb_lane,
7178 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7179 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
7180 ExtendLaneArg = true;
7181 [[fallthrough]];
7182 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
7183 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlallbt_lane,
7184 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7185 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
7186 ExtendLaneArg = true;
7187 [[fallthrough]];
7188 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
7189 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalltb_lane,
7190 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7191 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
7192 ExtendLaneArg = true;
7193 [[fallthrough]];
7194 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
7195 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalltt_lane,
7196 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7197 case NEON::BI__builtin_neon_vamin_f16:
7198 case NEON::BI__builtin_neon_vaminq_f16:
7199 case NEON::BI__builtin_neon_vamin_f32:
7200 case NEON::BI__builtin_neon_vaminq_f32:
7201 case NEON::BI__builtin_neon_vaminq_f64: {
7202 Int = Intrinsic::aarch64_neon_famin;
7203 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "famin");
7204 }
7205 case NEON::BI__builtin_neon_vamax_f16:
7206 case NEON::BI__builtin_neon_vamaxq_f16:
7207 case NEON::BI__builtin_neon_vamax_f32:
7208 case NEON::BI__builtin_neon_vamaxq_f32:
7209 case NEON::BI__builtin_neon_vamaxq_f64: {
7210 Int = Intrinsic::aarch64_neon_famax;
7211 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "famax");
7212 }
7213 case NEON::BI__builtin_neon_vscale_f16:
7214 case NEON::BI__builtin_neon_vscaleq_f16:
7215 case NEON::BI__builtin_neon_vscale_f32:
7216 case NEON::BI__builtin_neon_vscaleq_f32:
7217 case NEON::BI__builtin_neon_vscaleq_f64: {
7218 Int = Intrinsic::aarch64_neon_fp8_fscale;
7219 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "fscale");
7220 }
7221 }
7222}
7223
7224Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
7225 const CallExpr *E) {
7226 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
7227 BuiltinID == BPF::BI__builtin_btf_type_id ||
7228 BuiltinID == BPF::BI__builtin_preserve_type_info ||
7229 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
7230 "unexpected BPF builtin");
7231
7232 // A sequence number, injected into IR builtin functions, to
7233 // prevent CSE given the only difference of the function
7234 // may just be the debuginfo metadata.
7235 static uint32_t BuiltinSeqNum;
7236
7237 switch (BuiltinID) {
7238 default:
7239 llvm_unreachable("Unexpected BPF builtin");
7240 case BPF::BI__builtin_preserve_field_info: {
7241 const Expr *Arg = E->getArg(Arg: 0);
7242 bool IsBitField = Arg->IgnoreParens()->getObjectKind() == OK_BitField;
7243
7244 if (!getDebugInfo()) {
7245 CGM.Error(loc: E->getExprLoc(),
7246 error: "using __builtin_preserve_field_info() without -g");
7247 return IsBitField ? EmitLValue(E: Arg).getRawBitFieldPointer(CGF&: *this)
7248 : EmitLValue(E: Arg).emitRawPointer(CGF&: *this);
7249 }
7250
7251 // Enable underlying preserve_*_access_index() generation.
7252 bool OldIsInPreservedAIRegion = IsInPreservedAIRegion;
7253 IsInPreservedAIRegion = true;
7254 Value *FieldAddr = IsBitField ? EmitLValue(E: Arg).getRawBitFieldPointer(CGF&: *this)
7255 : EmitLValue(E: Arg).emitRawPointer(CGF&: *this);
7256 IsInPreservedAIRegion = OldIsInPreservedAIRegion;
7257
7258 ConstantInt *C = cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 1)));
7259 Value *InfoKind = ConstantInt::get(Ty: Int64Ty, V: C->getSExtValue());
7260
7261 // Built the IR for the preserve_field_info intrinsic.
7262 llvm::Function *FnGetFieldInfo = Intrinsic::getOrInsertDeclaration(
7263 M: &CGM.getModule(), id: Intrinsic::bpf_preserve_field_info,
7264 OverloadTys: {FieldAddr->getType()});
7265 return Builder.CreateCall(Callee: FnGetFieldInfo, Args: {FieldAddr, InfoKind});
7266 }
7267 case BPF::BI__builtin_btf_type_id:
7268 case BPF::BI__builtin_preserve_type_info: {
7269 if (!getDebugInfo()) {
7270 CGM.Error(loc: E->getExprLoc(), error: "using builtin function without -g");
7271 return nullptr;
7272 }
7273
7274 const Expr *Arg0 = E->getArg(Arg: 0);
7275 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
7276 Ty: Arg0->getType(), Loc: Arg0->getExprLoc());
7277
7278 ConstantInt *Flag = cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 1)));
7279 Value *FlagValue = ConstantInt::get(Ty: Int64Ty, V: Flag->getSExtValue());
7280 Value *SeqNumVal = ConstantInt::get(Ty: Int32Ty, V: BuiltinSeqNum++);
7281
7282 llvm::Function *FnDecl;
7283 if (BuiltinID == BPF::BI__builtin_btf_type_id)
7284 FnDecl = Intrinsic::getOrInsertDeclaration(
7285 M: &CGM.getModule(), id: Intrinsic::bpf_btf_type_id, OverloadTys: {});
7286 else
7287 FnDecl = Intrinsic::getOrInsertDeclaration(
7288 M: &CGM.getModule(), id: Intrinsic::bpf_preserve_type_info, OverloadTys: {});
7289 CallInst *Fn = Builder.CreateCall(Callee: FnDecl, Args: {SeqNumVal, FlagValue});
7290 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
7291 return Fn;
7292 }
7293 case BPF::BI__builtin_preserve_enum_value: {
7294 if (!getDebugInfo()) {
7295 CGM.Error(loc: E->getExprLoc(), error: "using builtin function without -g");
7296 return nullptr;
7297 }
7298
7299 const Expr *Arg0 = E->getArg(Arg: 0);
7300 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
7301 Ty: Arg0->getType(), Loc: Arg0->getExprLoc());
7302
7303 // Find enumerator
7304 const auto *UO = cast<UnaryOperator>(Val: Arg0->IgnoreParens());
7305 const auto *CE = cast<CStyleCastExpr>(Val: UO->getSubExpr());
7306 const auto *DR = cast<DeclRefExpr>(Val: CE->getSubExpr());
7307 const auto *Enumerator = cast<EnumConstantDecl>(Val: DR->getDecl());
7308
7309 auto InitVal = Enumerator->getInitVal();
7310 std::string InitValStr;
7311 if (InitVal.isNegative() || InitVal > uint64_t(INT64_MAX))
7312 InitValStr = std::to_string(val: InitVal.getSExtValue());
7313 else
7314 InitValStr = std::to_string(val: InitVal.getZExtValue());
7315 std::string EnumStr = Enumerator->getNameAsString() + ":" + InitValStr;
7316 Value *EnumStrVal = Builder.CreateGlobalString(Str: EnumStr);
7317
7318 ConstantInt *Flag = cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 1)));
7319 Value *FlagValue = ConstantInt::get(Ty: Int64Ty, V: Flag->getSExtValue());
7320 Value *SeqNumVal = ConstantInt::get(Ty: Int32Ty, V: BuiltinSeqNum++);
7321
7322 llvm::Function *IntrinsicFn = Intrinsic::getOrInsertDeclaration(
7323 M: &CGM.getModule(), id: Intrinsic::bpf_preserve_enum_value, OverloadTys: {});
7324 CallInst *Fn =
7325 Builder.CreateCall(Callee: IntrinsicFn, Args: {SeqNumVal, EnumStrVal, FlagValue});
7326 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
7327 return Fn;
7328 }
7329 }
7330}
7331
7332llvm::Value *CodeGenFunction::
7333BuildVector(ArrayRef<llvm::Value*> Ops) {
7334 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
7335 "Not a power-of-two sized vector!");
7336 bool AllConstants = true;
7337 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
7338 AllConstants &= isa<Constant>(Val: Ops[i]);
7339
7340 // If this is a constant vector, create a ConstantVector.
7341 if (AllConstants) {
7342 SmallVector<llvm::Constant*, 16> CstOps;
7343 for (llvm::Value *Op : Ops)
7344 CstOps.push_back(Elt: cast<Constant>(Val: Op));
7345 return llvm::ConstantVector::get(V: CstOps);
7346 }
7347
7348 // Otherwise, insertelement the values to build the vector.
7349 Value *Result = llvm::PoisonValue::get(
7350 T: llvm::FixedVectorType::get(ElementType: Ops[0]->getType(), NumElts: Ops.size()));
7351
7352 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
7353 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Ops[i], Idx: Builder.getInt64(C: i));
7354
7355 return Result;
7356}
7357
7358Value *CodeGenFunction::EmitAArch64CpuInit() {
7359 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
7360 llvm::FunctionCallee Func =
7361 CGM.CreateRuntimeFunction(Ty: FTy, Name: "__init_cpu_features_resolver");
7362 cast<llvm::GlobalValue>(Val: Func.getCallee())->setDSOLocal(true);
7363 cast<llvm::GlobalValue>(Val: Func.getCallee())
7364 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
7365 return Builder.CreateCall(Callee: Func);
7366}
7367
7368Value *CodeGenFunction::EmitAArch64CpuSupports(const CallExpr *E) {
7369 const Expr *ArgExpr = E->getArg(Arg: 0)->IgnoreParenCasts();
7370 StringRef ArgStr = cast<StringLiteral>(Val: ArgExpr)->getString();
7371 llvm::SmallVector<StringRef, 8> OrigFeatures;
7372 ArgStr.split(A&: OrigFeatures, Separator: "+");
7373 llvm::SmallVector<StringRef, 8> Features;
7374 for (StringRef Feature : OrigFeatures) {
7375 Feature = Feature.trim();
7376 if (!llvm::AArch64::parseFMVExtension(Extension: Feature))
7377 return Builder.getFalse();
7378 if (Feature != "default")
7379 Features.push_back(Elt: Feature);
7380 }
7381 return EmitAArch64CpuSupports(FeatureStrs: Features);
7382}
7383
7384llvm::Value *
7385CodeGenFunction::EmitAArch64CpuSupports(ArrayRef<StringRef> FeaturesStrs) {
7386 llvm::APInt FeaturesMask = llvm::AArch64::getCpuSupportsMask(Features: FeaturesStrs);
7387 Value *Result = Builder.getTrue();
7388 if (FeaturesMask != 0) {
7389 // Get features from structure in runtime library
7390 // struct {
7391 // unsigned long long features;
7392 // } __aarch64_cpu_features;
7393 llvm::Type *STy = llvm::StructType::get(elt1: Int64Ty);
7394 llvm::Constant *AArch64CPUFeatures =
7395 CGM.CreateRuntimeVariable(Ty: STy, Name: "__aarch64_cpu_features");
7396 cast<llvm::GlobalValue>(Val: AArch64CPUFeatures)->setDSOLocal(true);
7397 llvm::Value *CpuFeatures = Builder.CreateGEP(
7398 Ty: STy, Ptr: AArch64CPUFeatures,
7399 IdxList: {ConstantInt::get(Ty: Int32Ty, V: 0), ConstantInt::get(Ty: Int32Ty, V: 0)});
7400 Value *Features = Builder.CreateAlignedLoad(Ty: Int64Ty, Addr: CpuFeatures,
7401 Align: CharUnits::fromQuantity(Quantity: 8));
7402 Value *Mask = Builder.getInt(AI: FeaturesMask.trunc(width: 64));
7403 Value *Bitset = Builder.CreateAnd(LHS: Features, RHS: Mask);
7404 Value *Cmp = Builder.CreateICmpEQ(LHS: Bitset, RHS: Mask);
7405 Result = Builder.CreateAnd(LHS: Result, RHS: Cmp);
7406 }
7407 return Result;
7408}
7409