1//===---------- ARM.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfo.h"
14#include "CGBuiltin.h"
15#include "CGDebugInfo.h"
16#include "TargetInfo.h"
17#include "clang/Basic/AArch64CodeGenUtils.h"
18#include "clang/Basic/TargetBuiltins.h"
19#include "clang/CodeGenUtils/TargetUtils.h"
20#include "llvm/IR/InlineAsm.h"
21#include "llvm/IR/IntrinsicsAArch64.h"
22#include "llvm/IR/IntrinsicsARM.h"
23#include "llvm/IR/IntrinsicsBPF.h"
24#include "llvm/Support/AArch64MemoryHints.h"
25#include "llvm/TargetParser/AArch64TargetParser.h"
26
27#include <numeric>
28
29using namespace clang;
30using namespace CodeGen;
31using namespace llvm;
32using namespace clang::aarch64;
33
34static std::optional<CodeGenFunction::MSVCIntrin>
35translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
36 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
37 switch (BuiltinID) {
38 default:
39 return std::nullopt;
40 case clang::AArch64::BI_BitScanForward:
41 case clang::AArch64::BI_BitScanForward64:
42 return MSVCIntrin::_BitScanForward;
43 case clang::AArch64::BI_BitScanReverse:
44 case clang::AArch64::BI_BitScanReverse64:
45 return MSVCIntrin::_BitScanReverse;
46 case clang::AArch64::BI_InterlockedAnd64:
47 return MSVCIntrin::_InterlockedAnd;
48 case clang::AArch64::BI_InterlockedExchange64:
49 return MSVCIntrin::_InterlockedExchange;
50 case clang::AArch64::BI_InterlockedExchangeAdd64:
51 return MSVCIntrin::_InterlockedExchangeAdd;
52 case clang::AArch64::BI_InterlockedExchangeSub64:
53 return MSVCIntrin::_InterlockedExchangeSub;
54 case clang::AArch64::BI_InterlockedOr64:
55 return MSVCIntrin::_InterlockedOr;
56 case clang::AArch64::BI_InterlockedXor64:
57 return MSVCIntrin::_InterlockedXor;
58 case clang::AArch64::BI_InterlockedDecrement64:
59 return MSVCIntrin::_InterlockedDecrement;
60 case clang::AArch64::BI_InterlockedIncrement64:
61 return MSVCIntrin::_InterlockedIncrement;
62 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
63 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
64 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
65 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
66 return MSVCIntrin::_InterlockedExchangeAdd_acq;
67 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
68 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
69 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
70 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
71 return MSVCIntrin::_InterlockedExchangeAdd_rel;
72 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
73 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
74 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
75 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
76 return MSVCIntrin::_InterlockedExchangeAdd_nf;
77 case clang::AArch64::BI_InterlockedExchange8_acq:
78 case clang::AArch64::BI_InterlockedExchange16_acq:
79 case clang::AArch64::BI_InterlockedExchange_acq:
80 case clang::AArch64::BI_InterlockedExchange64_acq:
81 case clang::AArch64::BI_InterlockedExchangePointer_acq:
82 return MSVCIntrin::_InterlockedExchange_acq;
83 case clang::AArch64::BI_InterlockedExchange8_rel:
84 case clang::AArch64::BI_InterlockedExchange16_rel:
85 case clang::AArch64::BI_InterlockedExchange_rel:
86 case clang::AArch64::BI_InterlockedExchange64_rel:
87 case clang::AArch64::BI_InterlockedExchangePointer_rel:
88 return MSVCIntrin::_InterlockedExchange_rel;
89 case clang::AArch64::BI_InterlockedExchange8_nf:
90 case clang::AArch64::BI_InterlockedExchange16_nf:
91 case clang::AArch64::BI_InterlockedExchange_nf:
92 case clang::AArch64::BI_InterlockedExchange64_nf:
93 case clang::AArch64::BI_InterlockedExchangePointer_nf:
94 return MSVCIntrin::_InterlockedExchange_nf;
95 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
96 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
97 case clang::AArch64::BI_InterlockedCompareExchange_acq:
98 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
99 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
100 return MSVCIntrin::_InterlockedCompareExchange_acq;
101 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
102 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
103 case clang::AArch64::BI_InterlockedCompareExchange_rel:
104 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
105 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
106 return MSVCIntrin::_InterlockedCompareExchange_rel;
107 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
108 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
109 case clang::AArch64::BI_InterlockedCompareExchange_nf:
110 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
111 return MSVCIntrin::_InterlockedCompareExchange_nf;
112 case clang::AArch64::BI_InterlockedCompareExchange128:
113 return MSVCIntrin::_InterlockedCompareExchange128;
114 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
115 return MSVCIntrin::_InterlockedCompareExchange128_acq;
116 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
117 return MSVCIntrin::_InterlockedCompareExchange128_nf;
118 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
119 return MSVCIntrin::_InterlockedCompareExchange128_rel;
120 case clang::AArch64::BI_InterlockedOr8_acq:
121 case clang::AArch64::BI_InterlockedOr16_acq:
122 case clang::AArch64::BI_InterlockedOr_acq:
123 case clang::AArch64::BI_InterlockedOr64_acq:
124 return MSVCIntrin::_InterlockedOr_acq;
125 case clang::AArch64::BI_InterlockedOr8_rel:
126 case clang::AArch64::BI_InterlockedOr16_rel:
127 case clang::AArch64::BI_InterlockedOr_rel:
128 case clang::AArch64::BI_InterlockedOr64_rel:
129 return MSVCIntrin::_InterlockedOr_rel;
130 case clang::AArch64::BI_InterlockedOr8_nf:
131 case clang::AArch64::BI_InterlockedOr16_nf:
132 case clang::AArch64::BI_InterlockedOr_nf:
133 case clang::AArch64::BI_InterlockedOr64_nf:
134 return MSVCIntrin::_InterlockedOr_nf;
135 case clang::AArch64::BI_InterlockedXor8_acq:
136 case clang::AArch64::BI_InterlockedXor16_acq:
137 case clang::AArch64::BI_InterlockedXor_acq:
138 case clang::AArch64::BI_InterlockedXor64_acq:
139 return MSVCIntrin::_InterlockedXor_acq;
140 case clang::AArch64::BI_InterlockedXor8_rel:
141 case clang::AArch64::BI_InterlockedXor16_rel:
142 case clang::AArch64::BI_InterlockedXor_rel:
143 case clang::AArch64::BI_InterlockedXor64_rel:
144 return MSVCIntrin::_InterlockedXor_rel;
145 case clang::AArch64::BI_InterlockedXor8_nf:
146 case clang::AArch64::BI_InterlockedXor16_nf:
147 case clang::AArch64::BI_InterlockedXor_nf:
148 case clang::AArch64::BI_InterlockedXor64_nf:
149 return MSVCIntrin::_InterlockedXor_nf;
150 case clang::AArch64::BI_InterlockedAnd8_acq:
151 case clang::AArch64::BI_InterlockedAnd16_acq:
152 case clang::AArch64::BI_InterlockedAnd_acq:
153 case clang::AArch64::BI_InterlockedAnd64_acq:
154 return MSVCIntrin::_InterlockedAnd_acq;
155 case clang::AArch64::BI_InterlockedAnd8_rel:
156 case clang::AArch64::BI_InterlockedAnd16_rel:
157 case clang::AArch64::BI_InterlockedAnd_rel:
158 case clang::AArch64::BI_InterlockedAnd64_rel:
159 return MSVCIntrin::_InterlockedAnd_rel;
160 case clang::AArch64::BI_InterlockedAnd8_nf:
161 case clang::AArch64::BI_InterlockedAnd16_nf:
162 case clang::AArch64::BI_InterlockedAnd_nf:
163 case clang::AArch64::BI_InterlockedAnd64_nf:
164 return MSVCIntrin::_InterlockedAnd_nf;
165 case clang::AArch64::BI_InterlockedIncrement16_acq:
166 case clang::AArch64::BI_InterlockedIncrement_acq:
167 case clang::AArch64::BI_InterlockedIncrement64_acq:
168 return MSVCIntrin::_InterlockedIncrement_acq;
169 case clang::AArch64::BI_InterlockedIncrement16_rel:
170 case clang::AArch64::BI_InterlockedIncrement_rel:
171 case clang::AArch64::BI_InterlockedIncrement64_rel:
172 return MSVCIntrin::_InterlockedIncrement_rel;
173 case clang::AArch64::BI_InterlockedIncrement16_nf:
174 case clang::AArch64::BI_InterlockedIncrement_nf:
175 case clang::AArch64::BI_InterlockedIncrement64_nf:
176 return MSVCIntrin::_InterlockedIncrement_nf;
177 case clang::AArch64::BI_InterlockedDecrement16_acq:
178 case clang::AArch64::BI_InterlockedDecrement_acq:
179 case clang::AArch64::BI_InterlockedDecrement64_acq:
180 return MSVCIntrin::_InterlockedDecrement_acq;
181 case clang::AArch64::BI_InterlockedDecrement16_rel:
182 case clang::AArch64::BI_InterlockedDecrement_rel:
183 case clang::AArch64::BI_InterlockedDecrement64_rel:
184 return MSVCIntrin::_InterlockedDecrement_rel;
185 case clang::AArch64::BI_InterlockedDecrement16_nf:
186 case clang::AArch64::BI_InterlockedDecrement_nf:
187 case clang::AArch64::BI_InterlockedDecrement64_nf:
188 return MSVCIntrin::_InterlockedDecrement_nf;
189 }
190 llvm_unreachable("must return from switch");
191}
192
193static std::optional<CodeGenFunction::MSVCIntrin>
194translateArmToMsvcIntrin(unsigned BuiltinID) {
195 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
196 switch (BuiltinID) {
197 default:
198 return std::nullopt;
199 case clang::ARM::BI_BitScanForward:
200 case clang::ARM::BI_BitScanForward64:
201 return MSVCIntrin::_BitScanForward;
202 case clang::ARM::BI_BitScanReverse:
203 case clang::ARM::BI_BitScanReverse64:
204 return MSVCIntrin::_BitScanReverse;
205 case clang::ARM::BI_InterlockedAnd64:
206 return MSVCIntrin::_InterlockedAnd;
207 case clang::ARM::BI_InterlockedExchange64:
208 return MSVCIntrin::_InterlockedExchange;
209 case clang::ARM::BI_InterlockedExchangeAdd64:
210 return MSVCIntrin::_InterlockedExchangeAdd;
211 case clang::ARM::BI_InterlockedExchangeSub64:
212 return MSVCIntrin::_InterlockedExchangeSub;
213 case clang::ARM::BI_InterlockedOr64:
214 return MSVCIntrin::_InterlockedOr;
215 case clang::ARM::BI_InterlockedXor64:
216 return MSVCIntrin::_InterlockedXor;
217 case clang::ARM::BI_InterlockedDecrement64:
218 return MSVCIntrin::_InterlockedDecrement;
219 case clang::ARM::BI_InterlockedIncrement64:
220 return MSVCIntrin::_InterlockedIncrement;
221 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
222 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
223 case clang::ARM::BI_InterlockedExchangeAdd_acq:
224 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
225 return MSVCIntrin::_InterlockedExchangeAdd_acq;
226 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
227 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
228 case clang::ARM::BI_InterlockedExchangeAdd_rel:
229 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
230 return MSVCIntrin::_InterlockedExchangeAdd_rel;
231 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
232 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
233 case clang::ARM::BI_InterlockedExchangeAdd_nf:
234 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
235 return MSVCIntrin::_InterlockedExchangeAdd_nf;
236 case clang::ARM::BI_InterlockedExchange8_acq:
237 case clang::ARM::BI_InterlockedExchange16_acq:
238 case clang::ARM::BI_InterlockedExchange_acq:
239 case clang::ARM::BI_InterlockedExchange64_acq:
240 case clang::ARM::BI_InterlockedExchangePointer_acq:
241 return MSVCIntrin::_InterlockedExchange_acq;
242 case clang::ARM::BI_InterlockedExchange8_rel:
243 case clang::ARM::BI_InterlockedExchange16_rel:
244 case clang::ARM::BI_InterlockedExchange_rel:
245 case clang::ARM::BI_InterlockedExchange64_rel:
246 case clang::ARM::BI_InterlockedExchangePointer_rel:
247 return MSVCIntrin::_InterlockedExchange_rel;
248 case clang::ARM::BI_InterlockedExchange8_nf:
249 case clang::ARM::BI_InterlockedExchange16_nf:
250 case clang::ARM::BI_InterlockedExchange_nf:
251 case clang::ARM::BI_InterlockedExchange64_nf:
252 case clang::ARM::BI_InterlockedExchangePointer_nf:
253 return MSVCIntrin::_InterlockedExchange_nf;
254 case clang::ARM::BI_InterlockedCompareExchange8_acq:
255 case clang::ARM::BI_InterlockedCompareExchange16_acq:
256 case clang::ARM::BI_InterlockedCompareExchange_acq:
257 case clang::ARM::BI_InterlockedCompareExchange64_acq:
258 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
259 return MSVCIntrin::_InterlockedCompareExchange_acq;
260 case clang::ARM::BI_InterlockedCompareExchange8_rel:
261 case clang::ARM::BI_InterlockedCompareExchange16_rel:
262 case clang::ARM::BI_InterlockedCompareExchange_rel:
263 case clang::ARM::BI_InterlockedCompareExchange64_rel:
264 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
265 return MSVCIntrin::_InterlockedCompareExchange_rel;
266 case clang::ARM::BI_InterlockedCompareExchange8_nf:
267 case clang::ARM::BI_InterlockedCompareExchange16_nf:
268 case clang::ARM::BI_InterlockedCompareExchange_nf:
269 case clang::ARM::BI_InterlockedCompareExchange64_nf:
270 return MSVCIntrin::_InterlockedCompareExchange_nf;
271 case clang::ARM::BI_InterlockedOr8_acq:
272 case clang::ARM::BI_InterlockedOr16_acq:
273 case clang::ARM::BI_InterlockedOr_acq:
274 case clang::ARM::BI_InterlockedOr64_acq:
275 return MSVCIntrin::_InterlockedOr_acq;
276 case clang::ARM::BI_InterlockedOr8_rel:
277 case clang::ARM::BI_InterlockedOr16_rel:
278 case clang::ARM::BI_InterlockedOr_rel:
279 case clang::ARM::BI_InterlockedOr64_rel:
280 return MSVCIntrin::_InterlockedOr_rel;
281 case clang::ARM::BI_InterlockedOr8_nf:
282 case clang::ARM::BI_InterlockedOr16_nf:
283 case clang::ARM::BI_InterlockedOr_nf:
284 case clang::ARM::BI_InterlockedOr64_nf:
285 return MSVCIntrin::_InterlockedOr_nf;
286 case clang::ARM::BI_InterlockedXor8_acq:
287 case clang::ARM::BI_InterlockedXor16_acq:
288 case clang::ARM::BI_InterlockedXor_acq:
289 case clang::ARM::BI_InterlockedXor64_acq:
290 return MSVCIntrin::_InterlockedXor_acq;
291 case clang::ARM::BI_InterlockedXor8_rel:
292 case clang::ARM::BI_InterlockedXor16_rel:
293 case clang::ARM::BI_InterlockedXor_rel:
294 case clang::ARM::BI_InterlockedXor64_rel:
295 return MSVCIntrin::_InterlockedXor_rel;
296 case clang::ARM::BI_InterlockedXor8_nf:
297 case clang::ARM::BI_InterlockedXor16_nf:
298 case clang::ARM::BI_InterlockedXor_nf:
299 case clang::ARM::BI_InterlockedXor64_nf:
300 return MSVCIntrin::_InterlockedXor_nf;
301 case clang::ARM::BI_InterlockedAnd8_acq:
302 case clang::ARM::BI_InterlockedAnd16_acq:
303 case clang::ARM::BI_InterlockedAnd_acq:
304 case clang::ARM::BI_InterlockedAnd64_acq:
305 return MSVCIntrin::_InterlockedAnd_acq;
306 case clang::ARM::BI_InterlockedAnd8_rel:
307 case clang::ARM::BI_InterlockedAnd16_rel:
308 case clang::ARM::BI_InterlockedAnd_rel:
309 case clang::ARM::BI_InterlockedAnd64_rel:
310 return MSVCIntrin::_InterlockedAnd_rel;
311 case clang::ARM::BI_InterlockedAnd8_nf:
312 case clang::ARM::BI_InterlockedAnd16_nf:
313 case clang::ARM::BI_InterlockedAnd_nf:
314 case clang::ARM::BI_InterlockedAnd64_nf:
315 return MSVCIntrin::_InterlockedAnd_nf;
316 case clang::ARM::BI_InterlockedIncrement16_acq:
317 case clang::ARM::BI_InterlockedIncrement_acq:
318 case clang::ARM::BI_InterlockedIncrement64_acq:
319 return MSVCIntrin::_InterlockedIncrement_acq;
320 case clang::ARM::BI_InterlockedIncrement16_rel:
321 case clang::ARM::BI_InterlockedIncrement_rel:
322 case clang::ARM::BI_InterlockedIncrement64_rel:
323 return MSVCIntrin::_InterlockedIncrement_rel;
324 case clang::ARM::BI_InterlockedIncrement16_nf:
325 case clang::ARM::BI_InterlockedIncrement_nf:
326 case clang::ARM::BI_InterlockedIncrement64_nf:
327 return MSVCIntrin::_InterlockedIncrement_nf;
328 case clang::ARM::BI_InterlockedDecrement16_acq:
329 case clang::ARM::BI_InterlockedDecrement_acq:
330 case clang::ARM::BI_InterlockedDecrement64_acq:
331 return MSVCIntrin::_InterlockedDecrement_acq;
332 case clang::ARM::BI_InterlockedDecrement16_rel:
333 case clang::ARM::BI_InterlockedDecrement_rel:
334 case clang::ARM::BI_InterlockedDecrement64_rel:
335 return MSVCIntrin::_InterlockedDecrement_rel;
336 case clang::ARM::BI_InterlockedDecrement16_nf:
337 case clang::ARM::BI_InterlockedDecrement_nf:
338 case clang::ARM::BI_InterlockedDecrement64_nf:
339 return MSVCIntrin::_InterlockedDecrement_nf;
340 }
341 llvm_unreachable("must return from switch");
342}
343
344// Emit an intrinsic where all operands are of the same type as the result.
345// Depending on mode, this may be a constrained floating-point intrinsic.
346static Value *emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
347 unsigned IntrinsicID,
348 unsigned ConstrainedIntrinsicID,
349 llvm::Type *Ty,
350 ArrayRef<Value *> Args) {
351 Function *F;
352 if (CGF.Builder.getIsFPConstrained())
353 F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Ty);
354 else
355 F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Ty);
356
357 if (CGF.Builder.getIsFPConstrained())
358 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args);
359
360 return CGF.Builder.CreateCall(Callee: F, Args);
361}
362
363static llvm::FixedVectorType *GetNeonType(CodeGenFunction *CGF,
364 NeonTypeFlags TypeFlags,
365 bool HasFastHalfType = true,
366 bool V1Ty = false,
367 bool AllowBFloatArgsAndRet = true) {
368 int IsQuad = TypeFlags.isQuad();
369 switch (TypeFlags.getEltType()) {
370 case NeonTypeFlags::Int8:
371 case NeonTypeFlags::Poly8:
372 case NeonTypeFlags::MFloat8:
373 return llvm::FixedVectorType::get(ElementType: CGF->Int8Ty, NumElts: V1Ty ? 1 : (8 << IsQuad));
374 case NeonTypeFlags::Int16:
375 case NeonTypeFlags::Poly16:
376 return llvm::FixedVectorType::get(ElementType: CGF->Int16Ty, NumElts: V1Ty ? 1 : (4 << IsQuad));
377 case NeonTypeFlags::BFloat16:
378 if (AllowBFloatArgsAndRet)
379 return llvm::FixedVectorType::get(ElementType: CGF->BFloatTy, NumElts: V1Ty ? 1 : (4 << IsQuad));
380 return llvm::FixedVectorType::get(ElementType: CGF->Int16Ty, NumElts: V1Ty ? 1 : (4 << IsQuad));
381 case NeonTypeFlags::Float16:
382 if (HasFastHalfType)
383 return llvm::FixedVectorType::get(ElementType: CGF->HalfTy, NumElts: V1Ty ? 1 : (4 << IsQuad));
384 return llvm::FixedVectorType::get(ElementType: CGF->Int16Ty, NumElts: V1Ty ? 1 : (4 << IsQuad));
385 case NeonTypeFlags::Int32:
386 return llvm::FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: V1Ty ? 1 : (2 << IsQuad));
387 case NeonTypeFlags::Int64:
388 case NeonTypeFlags::Poly64:
389 return llvm::FixedVectorType::get(ElementType: CGF->Int64Ty, NumElts: V1Ty ? 1 : (1 << IsQuad));
390 case NeonTypeFlags::Poly128:
391 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
392 // There is a lot of i128 and f128 API missing.
393 // so we use v16i8 to represent poly128 and get pattern matched.
394 return llvm::FixedVectorType::get(ElementType: CGF->Int8Ty, NumElts: 16);
395 case NeonTypeFlags::Float32:
396 return llvm::FixedVectorType::get(ElementType: CGF->FloatTy, NumElts: V1Ty ? 1 : (2 << IsQuad));
397 case NeonTypeFlags::Float64:
398 return llvm::FixedVectorType::get(ElementType: CGF->DoubleTy, NumElts: V1Ty ? 1 : (1 << IsQuad));
399 }
400 llvm_unreachable("Unknown vector element type!");
401}
402
403static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
404 NeonTypeFlags IntTypeFlags) {
405 int IsQuad = IntTypeFlags.isQuad();
406 switch (IntTypeFlags.getEltType()) {
407 case NeonTypeFlags::Int16:
408 return llvm::FixedVectorType::get(ElementType: CGF->HalfTy, NumElts: (4 << IsQuad));
409 case NeonTypeFlags::Int32:
410 return llvm::FixedVectorType::get(ElementType: CGF->FloatTy, NumElts: (2 << IsQuad));
411 case NeonTypeFlags::Int64:
412 return llvm::FixedVectorType::get(ElementType: CGF->DoubleTy, NumElts: (1 << IsQuad));
413 default:
414 llvm_unreachable("Type can't be converted to floating-point!");
415 }
416}
417
418Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C,
419 const ElementCount &Count) {
420 Value *SV = llvm::ConstantVector::getSplat(EC: Count, Elt: C);
421 return Builder.CreateShuffleVector(V1: V, V2: V, Mask: SV, Name: "lane");
422}
423
424Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
425 ElementCount EC = cast<llvm::VectorType>(Val: V->getType())->getElementCount();
426 return EmitNeonSplat(V, C, Count: EC);
427}
428
429Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
430 const char *name,
431 unsigned shift, bool rightshift) {
432 unsigned j = 0;
433 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
434 ai != ae; ++ai, ++j) {
435 if (F->isConstrainedFPIntrinsic())
436 if (ai->getType()->isMetadataTy())
437 continue;
438 if (shift > 0 && shift == j)
439 Ops[j] = EmitNeonShiftVector(V: Ops[j], Ty: ai->getType(), negateForRightShift: rightshift);
440 else
441 Ops[j] = Builder.CreateBitCast(V: Ops[j], DestTy: ai->getType(), Name: name);
442 }
443
444 if (F->isConstrainedFPIntrinsic())
445 return Builder.CreateConstrainedFPCall(Callee: F, Args: Ops, Name: name);
446 return Builder.CreateCall(Callee: F, Args: Ops, Name: name);
447}
448
449Value *CodeGenFunction::EmitFP8NeonCall(unsigned IID,
450 ArrayRef<llvm::Type *> Tys,
451 SmallVectorImpl<Value *> &Ops,
452 const CallExpr *E, const char *name) {
453 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_set_fpmr),
454 Args: Ops.pop_back_val());
455 return EmitNeonCall(F: CGM.getIntrinsic(IID, Tys), Ops, name);
456}
457
458llvm::Value *CodeGenFunction::EmitFP8NeonFDOTCall(
459 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
460 SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
461
462 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
463 RetTy->getPrimitiveSizeInBits();
464 llvm::Type *Tys[] = {llvm::FixedVectorType::get(ElementType: RetTy, NumElts: ElemCount),
465 Ops[1]->getType()};
466 if (ExtendLaneArg) {
467 auto *VT = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16);
468 Ops[2] = Builder.CreateInsertVector(DstType: VT, SrcVec: PoisonValue::get(T: VT), SubVec: Ops[2],
469 Idx: uint64_t(0));
470 }
471 return EmitFP8NeonCall(IID, Tys, Ops, E, name);
472}
473
474llvm::Value *CodeGenFunction::EmitFP8NeonFMLACall(
475 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
476 SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
477
478 if (ExtendLaneArg) {
479 auto *VT = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16);
480 Ops[2] = Builder.CreateInsertVector(DstType: VT, SrcVec: PoisonValue::get(T: VT), SubVec: Ops[2],
481 Idx: uint64_t(0));
482 }
483 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
484 RetTy->getPrimitiveSizeInBits();
485 return EmitFP8NeonCall(IID, Tys: {llvm::FixedVectorType::get(ElementType: RetTy, NumElts: ElemCount)},
486 Ops, E, name);
487}
488
489Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
490 bool neg) {
491 int SV = cast<ConstantInt>(Val: V)->getSExtValue();
492 return ConstantInt::getSigned(Ty, V: neg ? -SV : SV);
493}
494
495Value *CodeGenFunction::EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
496 llvm::Type *Ty1, bool Extract,
497 SmallVectorImpl<llvm::Value *> &Ops,
498 const CallExpr *E,
499 const char *name) {
500 llvm::Type *Tys[] = {Ty0, Ty1};
501 if (Extract) {
502 // Op[0] is mfloat8x16_t, but the intrinsic converts only the lower part of
503 // the vector.
504 Tys[1] = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 8);
505 Ops[0] = Builder.CreateExtractVector(DstType: Tys[1], SrcVec: Ops[0], Idx: uint64_t(0));
506 }
507 return EmitFP8NeonCall(IID, Tys, Ops, E, name);
508}
509
510// Right-shift a vector by a constant.
511Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
512 llvm::Type *Ty, bool usgn,
513 const char *name) {
514 llvm::VectorType *VTy = cast<llvm::VectorType>(Val: Ty);
515
516 int ShiftAmt = cast<ConstantInt>(Val: Shift)->getSExtValue();
517 int EltSize = VTy->getScalarSizeInBits();
518
519 Vec = Builder.CreateBitCast(V: Vec, DestTy: Ty);
520
521 // lshr/ashr are undefined when the shift amount is equal to the vector
522 // element size.
523 if (ShiftAmt == EltSize) {
524 if (usgn) {
525 // Right-shifting an unsigned value by its size yields 0.
526 return llvm::ConstantAggregateZero::get(Ty: VTy);
527 } else {
528 // Right-shifting a signed value by its size is equivalent
529 // to a shift of size-1.
530 --ShiftAmt;
531 Shift = ConstantInt::get(Ty: VTy->getElementType(), V: ShiftAmt);
532 }
533 }
534
535 Shift = EmitNeonShiftVector(V: Shift, Ty, neg: false);
536 if (usgn)
537 return Builder.CreateLShr(LHS: Vec, RHS: Shift, Name: name);
538 return Builder.CreateAShr(LHS: Vec, RHS: Shift, Name: name);
539}
540
541// clang-format off
542static const ARMNeonVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
543 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
544 NEONMAP0(splat_lane_v),
545 NEONMAP0(splat_laneq_v),
546 NEONMAP0(splatq_lane_v),
547 NEONMAP0(splatq_laneq_v),
548 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
549 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
550 NEONMAP1(vabs_v, arm_neon_vabs, 0),
551 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
552 NEONMAP0(vadd_v),
553 NEONMAP0(vaddhn_v),
554 NEONMAP0(vaddq_v),
555 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
556 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
557 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
558 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
559 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
560 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
561 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
562 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
563 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
564 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
565 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
566 NEONMAP1(vcadd_rot270_f16, arm_neon_vcadd_rot270, Add1ArgType),
567 NEONMAP1(vcadd_rot270_f32, arm_neon_vcadd_rot270, Add1ArgType),
568 NEONMAP1(vcadd_rot90_f16, arm_neon_vcadd_rot90, Add1ArgType),
569 NEONMAP1(vcadd_rot90_f32, arm_neon_vcadd_rot90, Add1ArgType),
570 NEONMAP1(vcaddq_rot270_f16, arm_neon_vcadd_rot270, Add1ArgType),
571 NEONMAP1(vcaddq_rot270_f32, arm_neon_vcadd_rot270, Add1ArgType),
572 NEONMAP1(vcaddq_rot270_f64, arm_neon_vcadd_rot270, Add1ArgType),
573 NEONMAP1(vcaddq_rot90_f16, arm_neon_vcadd_rot90, Add1ArgType),
574 NEONMAP1(vcaddq_rot90_f32, arm_neon_vcadd_rot90, Add1ArgType),
575 NEONMAP1(vcaddq_rot90_f64, arm_neon_vcadd_rot90, Add1ArgType),
576 NEONMAP1(vcage_v, arm_neon_vacge, 0),
577 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
578 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
579 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
580 NEONMAP1(vcale_v, arm_neon_vacge, 0),
581 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
582 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
583 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
584 NEONMAP0(vceqz_v),
585 NEONMAP0(vceqzq_v),
586 NEONMAP0(vcgez_v),
587 NEONMAP0(vcgezq_v),
588 NEONMAP0(vcgtz_v),
589 NEONMAP0(vcgtzq_v),
590 NEONMAP0(vclez_v),
591 NEONMAP0(vclezq_v),
592 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
593 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
594 NEONMAP0(vcltz_v),
595 NEONMAP0(vcltzq_v),
596 NEONMAP1(vclz_v, ctlz, Add1ArgType),
597 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
598 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
599 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
600 NEONMAP0(vcvt_f16_s16),
601 NEONMAP0(vcvt_f16_u16),
602 NEONMAP0(vcvt_f32_v),
603 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
604 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
605 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
606 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
607 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
608 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
609 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
610 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
611 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
612 NEONMAP0(vcvt_s16_f16),
613 NEONMAP0(vcvt_s32_v),
614 NEONMAP0(vcvt_s64_v),
615 NEONMAP0(vcvt_u16_f16),
616 NEONMAP0(vcvt_u32_v),
617 NEONMAP0(vcvt_u64_v),
618 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
619 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
620 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
621 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
622 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
623 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
624 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
625 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
626 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
627 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
628 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
629 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
630 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
631 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
632 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
633 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
634 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
635 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
636 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
637 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
638 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
639 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
640 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
641 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
642 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
643 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
644 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
645 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
646 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
647 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
648 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
649 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
650 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
651 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
652 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
653 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
654 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
655 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
656 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
657 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
658 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
659 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
660 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
661 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
662 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
663 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
664 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
665 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
666 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
667 NEONMAP0(vcvtq_f16_s16),
668 NEONMAP0(vcvtq_f16_u16),
669 NEONMAP0(vcvtq_f32_v),
670 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
671 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
672 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
673 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
674 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
675 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
676 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
677 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
678 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
679 NEONMAP0(vcvtq_s16_f16),
680 NEONMAP0(vcvtq_s32_v),
681 NEONMAP0(vcvtq_s64_v),
682 NEONMAP0(vcvtq_u16_f16),
683 NEONMAP0(vcvtq_u32_v),
684 NEONMAP0(vcvtq_u64_v),
685 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
686 NEONMAP1(vdot_u32, arm_neon_udot, 0),
687 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
688 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
689 NEONMAP0(vext_v),
690 NEONMAP0(vextq_v),
691 NEONMAP0(vfma_v),
692 NEONMAP0(vfmaq_v),
693 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
694 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
695 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
696 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
697 NEONMAP0(vld1_dup_v),
698 NEONMAP1(vld1_v, arm_neon_vld1, 0),
699 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
700 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
701 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
702 NEONMAP0(vld1q_dup_v),
703 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
704 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
705 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
706 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
707 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
708 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
709 NEONMAP1(vld2_v, arm_neon_vld2, 0),
710 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
711 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
712 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
713 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
714 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
715 NEONMAP1(vld3_v, arm_neon_vld3, 0),
716 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
717 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
718 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
719 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
720 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
721 NEONMAP1(vld4_v, arm_neon_vld4, 0),
722 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
723 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
724 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
725 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
726 NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
727 NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
728 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
729 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
730 NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
731 NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
732 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
733 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
734 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
735 NEONMAP0(vmovl_v),
736 NEONMAP0(vmovn_v),
737 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
738 NEONMAP0(vmull_v),
739 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
740 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
741 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
742 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
743 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
744 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
745 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
746 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
747 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
748 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
749 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
750 NEONMAP2(vqadd_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
751 NEONMAP2(vqaddq_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
752 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
753 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
754 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
755 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
756 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
757 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
758 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
759 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
760 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
761 NEONMAP1(vqrdmlah_s16, arm_neon_vqrdmlah, Add1ArgType),
762 NEONMAP1(vqrdmlah_s32, arm_neon_vqrdmlah, Add1ArgType),
763 NEONMAP1(vqrdmlahq_s16, arm_neon_vqrdmlah, Add1ArgType),
764 NEONMAP1(vqrdmlahq_s32, arm_neon_vqrdmlah, Add1ArgType),
765 NEONMAP1(vqrdmlsh_s16, arm_neon_vqrdmlsh, Add1ArgType),
766 NEONMAP1(vqrdmlsh_s32, arm_neon_vqrdmlsh, Add1ArgType),
767 NEONMAP1(vqrdmlshq_s16, arm_neon_vqrdmlsh, Add1ArgType),
768 NEONMAP1(vqrdmlshq_s32, arm_neon_vqrdmlsh, Add1ArgType),
769 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
770 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
771 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
772 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
773 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
774 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
775 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
776 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
777 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
778 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
779 NEONMAP2(vqsub_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
780 NEONMAP2(vqsubq_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
781 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
782 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
783 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
784 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
785 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
786 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
787 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
788 NEONMAP1(vrnd_v, trunc, Add1ArgType),
789 NEONMAP1(vrnda_v, round, Add1ArgType),
790 NEONMAP1(vrndaq_v, round, Add1ArgType),
791 NEONMAP0(vrndi_v),
792 NEONMAP0(vrndiq_v),
793 NEONMAP1(vrndm_v, floor, Add1ArgType),
794 NEONMAP1(vrndmq_v, floor, Add1ArgType),
795 NEONMAP1(vrndn_v, roundeven, Add1ArgType),
796 NEONMAP1(vrndnq_v, roundeven, Add1ArgType),
797 NEONMAP1(vrndp_v, ceil, Add1ArgType),
798 NEONMAP1(vrndpq_v, ceil, Add1ArgType),
799 NEONMAP1(vrndq_v, trunc, Add1ArgType),
800 NEONMAP1(vrndx_v, rint, Add1ArgType),
801 NEONMAP1(vrndxq_v, rint, Add1ArgType),
802 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
803 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
804 NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
805 NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
806 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
807 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
808 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
809 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
810 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
811 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
812 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
813 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
814 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
815 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
816 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
817 NEONMAP0(vshl_n_v),
818 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
819 NEONMAP0(vshll_n_v),
820 NEONMAP0(vshlq_n_v),
821 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
822 NEONMAP0(vshr_n_v),
823 NEONMAP0(vshrn_n_v),
824 NEONMAP0(vshrq_n_v),
825 NEONMAP1(vst1_v, arm_neon_vst1, 0),
826 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
827 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
828 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
829 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
830 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
831 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
832 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
833 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
834 NEONMAP1(vst2_v, arm_neon_vst2, 0),
835 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
836 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
837 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
838 NEONMAP1(vst3_v, arm_neon_vst3, 0),
839 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
840 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
841 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
842 NEONMAP1(vst4_v, arm_neon_vst4, 0),
843 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
844 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
845 NEONMAP0(vsubhn_v),
846 NEONMAP0(vtrn_v),
847 NEONMAP0(vtrnq_v),
848 NEONMAP0(vtst_v),
849 NEONMAP0(vtstq_v),
850 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
851 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
852 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
853 NEONMAP0(vuzp_v),
854 NEONMAP0(vuzpq_v),
855 NEONMAP0(vzip_v),
856 NEONMAP0(vzipq_v)
857};
858
859// clang-format on
860
861// Some intrinsics are equivalent for codegen.
862static const std::pair<unsigned, unsigned> NEONEquivalentIntrinsicMap[] = {
863 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
864 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
865 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
866 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
867 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
868 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
869 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
870 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
871 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
872 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
873 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
874 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
875 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
876 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
877 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
878 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
879 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
880 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
881 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
882 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
883 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
884 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
885 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
886 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
887 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
888 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
889 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
890 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
891 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
892 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
893 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
894 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
895 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
896 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
897 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
898 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
899 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
900 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
901 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
902 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
903 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
904 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
905 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
906 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
907 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
908 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
909 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
910 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
911 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
912 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
913 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
914 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
915 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
916 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
917 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
918 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
919 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
920 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
921 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
922 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
923 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
924 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
925 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
926 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
927 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
928 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
929 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
930 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
931 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
932 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
933 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
934 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
935 // The mangling rules cause us to have one ID for each type for vldap1(q)_lane
936 // and vstl1(q)_lane, but codegen is equivalent for all of them. Choose an
937 // arbitrary one to be handled as tha canonical variation.
938 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
939 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
940 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
941 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
942 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
943 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
944 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
945 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
946 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
947 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
948 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
949 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
950};
951
952#undef NEONMAP0
953#undef NEONMAP1
954#undef NEONMAP2
955
956#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
957 {SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
958
959#define SVEMAP2(NameBase, TypeModifier) \
960 {SVE::BI__builtin_sve_##NameBase, 0, TypeModifier}
961static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SVEIntrinsicMap[] = {
962#define GET_SVE_LLVM_INTRINSIC_MAP
963#include "clang/Basic/arm_sve_builtin_cg.inc"
964#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
965#undef GET_SVE_LLVM_INTRINSIC_MAP
966};
967
968#undef SVEMAP1
969#undef SVEMAP2
970
971#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
972 {SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
973
974#define SMEMAP2(NameBase, TypeModifier) \
975 {SME::BI__builtin_sme_##NameBase, 0, TypeModifier}
976static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SMEIntrinsicMap[] = {
977#define GET_SME_LLVM_INTRINSIC_MAP
978#include "clang/Basic/arm_sme_builtin_cg.inc"
979#undef GET_SME_LLVM_INTRINSIC_MAP
980};
981
982#undef SMEMAP1
983#undef SMEMAP2
984
985static bool NEONSIMDIntrinsicsProvenSorted = false;
986
987static bool AArch64SIMDIntrinsicsProvenSorted = false;
988static bool AArch64SISDIntrinsicsProvenSorted = false;
989static bool AArch64SVEIntrinsicsProvenSorted = false;
990static bool AArch64SMEIntrinsicsProvenSorted = false;
991
992// Check if Builtin `BuiltinId` is present in `IntrinsicMap`. If yes, returns
993// the corresponding info struct.
994template <typename IntrinsicInfo>
995static const IntrinsicInfo *
996findARMVectorIntrinsicInMap(ArrayRef<IntrinsicInfo> IntrinsicMap,
997 unsigned BuiltinID, bool &MapProvenSorted) {
998
999#ifndef NDEBUG
1000 if (!MapProvenSorted) {
1001 assert(llvm::is_sorted(IntrinsicMap));
1002 MapProvenSorted = true;
1003 }
1004#endif
1005
1006 const IntrinsicInfo *Builtin = llvm::lower_bound(IntrinsicMap, BuiltinID);
1007
1008 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
1009 return Builtin;
1010
1011 return nullptr;
1012}
1013
1014Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
1015 unsigned Modifier,
1016 llvm::Type *ArgType,
1017 const CallExpr *E) {
1018 int VectorSize = 0;
1019 if (Modifier & Use64BitVectors)
1020 VectorSize = 64;
1021 else if (Modifier & Use128BitVectors)
1022 VectorSize = 128;
1023
1024 // Return type.
1025 SmallVector<llvm::Type *, 3> Tys;
1026 if (Modifier & AddRetType) {
1027 llvm::Type *Ty = ConvertType(T: E->getCallReturnType(Ctx: getContext()));
1028 if (Modifier & VectorizeRetType)
1029 Ty = llvm::FixedVectorType::get(
1030 ElementType: Ty, NumElts: VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
1031
1032 Tys.push_back(Elt: Ty);
1033 }
1034
1035 // Arguments.
1036 if (Modifier & VectorizeArgTypes) {
1037 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
1038 ArgType = llvm::FixedVectorType::get(ElementType: ArgType, NumElts: Elts);
1039 }
1040
1041 if (Modifier & (Add1ArgType | Add2ArgTypes))
1042 Tys.push_back(Elt: ArgType);
1043
1044 if (Modifier & Add2ArgTypes)
1045 Tys.push_back(Elt: ArgType);
1046
1047 if (Modifier & InventFloatType)
1048 Tys.push_back(Elt: FloatTy);
1049
1050 return CGM.getIntrinsic(IID: IntrinsicID, Tys);
1051}
1052
1053//===----------------------------------------------------------------------===//
1054// Emit-helpers
1055//===----------------------------------------------------------------------===//
1056static Value *EmitCommonNeonSISDBuiltinExpr(
1057 CodeGenFunction &CGF, const ARMNeonVectorIntrinsicInfo &SISDInfo,
1058 SmallVectorImpl<Value *> &Ops, const CallExpr *E) {
1059 assert(SISDInfo.LLVMIntrinsic && "Generic code assumes a valid intrinsic");
1060
1061 switch (SISDInfo.BuiltinID) {
1062 case NEON::BI__builtin_neon_vcled_s64:
1063 case NEON::BI__builtin_neon_vcled_u64:
1064 case NEON::BI__builtin_neon_vcles_f32:
1065 case NEON::BI__builtin_neon_vcled_f64:
1066 case NEON::BI__builtin_neon_vcltd_s64:
1067 case NEON::BI__builtin_neon_vcltd_u64:
1068 case NEON::BI__builtin_neon_vclts_f32:
1069 case NEON::BI__builtin_neon_vcltd_f64:
1070 case NEON::BI__builtin_neon_vcales_f32:
1071 case NEON::BI__builtin_neon_vcaled_f64:
1072 case NEON::BI__builtin_neon_vcalts_f32:
1073 case NEON::BI__builtin_neon_vcaltd_f64:
1074 // Only one direction of comparisons actually exist, cmle is actually a cmge
1075 // with swapped operands. The table gives us the right intrinsic but we
1076 // still need to do the swap.
1077 std::swap(a&: Ops[0], b&: Ops[1]);
1078 break;
1079 }
1080
1081 // Use fptosi.sat/fptoui.sat unless under strict FP.
1082 unsigned LLVMIntrinsic = SISDInfo.LLVMIntrinsic;
1083 if (!CGF.Builder.getIsFPConstrained()) {
1084 if (LLVMIntrinsic == Intrinsic::aarch64_neon_fcvtzs)
1085 LLVMIntrinsic = Intrinsic::fptosi_sat;
1086 else if (LLVMIntrinsic == Intrinsic::aarch64_neon_fcvtzu)
1087 LLVMIntrinsic = Intrinsic::fptoui_sat;
1088 }
1089 llvm::Type *ArgTy = CGF.ConvertType(T: E->getArg(Arg: 0)->getType());
1090 Function *F = CGF.LookupNeonLLVMIntrinsic(IntrinsicID: LLVMIntrinsic,
1091 Modifier: SISDInfo.TypeModifier, ArgType: ArgTy, E);
1092
1093 int j = 0;
1094 ConstantInt *C0 = ConstantInt::get(Ty: CGF.SizeTy, V: 0);
1095 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
1096 ai != ae; ++ai, ++j) {
1097 llvm::Type *ArgTy = ai->getType();
1098 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
1099 ArgTy->getPrimitiveSizeInBits())
1100 continue;
1101 assert(
1102 ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy() &&
1103 "Expecting vector LLVM intrinsic type and scalar Clang builtin type!");
1104
1105 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
1106 // it before inserting.
1107 Ops[j] = CGF.Builder.CreateTruncOrBitCast(
1108 V: Ops[j], DestTy: cast<llvm::VectorType>(Val: ArgTy)->getElementType());
1109 Ops[j] =
1110 CGF.Builder.CreateInsertElement(Vec: PoisonValue::get(T: ArgTy), NewElt: Ops[j], Idx: C0);
1111 }
1112
1113 Value *Result = CGF.EmitNeonCall(F, Ops, name: SISDInfo.NameHint);
1114 llvm::Type *ResultType = CGF.ConvertType(T: E->getType());
1115 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
1116 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
1117 return CGF.Builder.CreateExtractElement(Vec: Result, Idx: C0);
1118
1119 return CGF.Builder.CreateBitCast(V: Result, DestTy: ResultType, Name: SISDInfo.NameHint);
1120}
1121
1122Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
1123 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
1124 const char *NameHint, unsigned Modifier, const CallExpr *E,
1125 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1,
1126 llvm::Triple::ArchType Arch) {
1127
1128 // Extract the trailing immediate argument that encodes the type discriminator
1129 // for this overloaded intrinsic.
1130 // TODO: Move to the parent code that takes care of argument processing.
1131 const Expr *Arg = E->getArg(Arg: E->getNumArgs() - 1);
1132 std::optional<llvm::APSInt> NeonTypeConst =
1133 Arg->getIntegerConstantExpr(Ctx: getContext());
1134 if (!NeonTypeConst)
1135 return nullptr;
1136
1137 // Determine the type of this overloaded NEON intrinsic.
1138 NeonTypeFlags Type(NeonTypeConst->getZExtValue());
1139 const bool Usgn = Type.isUnsigned();
1140 const bool Quad = Type.isQuad();
1141 const bool Floating = Type.isFloatingPoint();
1142 const bool HasFastHalfType = getTarget().hasFastHalfType();
1143 const bool AllowBFloatArgsAndRet =
1144 getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
1145
1146 llvm::FixedVectorType *VTy =
1147 GetNeonType(CGF: this, TypeFlags: Type, HasFastHalfType, V1Ty: false, AllowBFloatArgsAndRet);
1148 llvm::Type *Ty = VTy;
1149 if (!Ty)
1150 return nullptr;
1151
1152 auto getAlignmentValue32 = [&](Address addr) -> Value* {
1153 return Builder.getInt32(C: addr.getAlignment().getQuantity());
1154 };
1155
1156 unsigned Int = LLVMIntrinsic;
1157 if ((Modifier & UnsignedAlts) && !Usgn)
1158 Int = AltLLVMIntrinsic;
1159
1160 switch (BuiltinID) {
1161 default: break;
1162 case NEON::BI__builtin_neon_splat_lane_v:
1163 case NEON::BI__builtin_neon_splat_laneq_v:
1164 case NEON::BI__builtin_neon_splatq_lane_v:
1165 case NEON::BI__builtin_neon_splatq_laneq_v: {
1166 auto NumElements = VTy->getElementCount();
1167 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
1168 NumElements = NumElements * 2;
1169 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
1170 NumElements = NumElements.divideCoefficientBy(RHS: 2);
1171
1172 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: VTy);
1173 return EmitNeonSplat(V: Ops[0], C: cast<ConstantInt>(Val: Ops[1]), Count: NumElements);
1174 }
1175 case NEON::BI__builtin_neon_vpadd_v:
1176 case NEON::BI__builtin_neon_vpaddq_v:
1177 // We don't allow fp/int overloading of intrinsics.
1178 if (VTy->getElementType()->isFloatingPointTy() &&
1179 Int == Intrinsic::aarch64_neon_addp)
1180 Int = Intrinsic::aarch64_neon_faddp;
1181 break;
1182 case NEON::BI__builtin_neon_vabs_v:
1183 case NEON::BI__builtin_neon_vabsq_v:
1184 if (VTy->getElementType()->isFloatingPointTy())
1185 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::fabs, Tys: Ty), Ops, name: "vabs");
1186 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys: Ty), Ops, name: "vabs");
1187 case NEON::BI__builtin_neon_vadd_v:
1188 case NEON::BI__builtin_neon_vaddq_v: {
1189 llvm::Type *VTy = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Quad ? 16 : 8);
1190 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: VTy);
1191 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: VTy);
1192 Ops[0] = Builder.CreateXor(LHS: Ops[0], RHS: Ops[1]);
1193 return Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1194 }
1195 case NEON::BI__builtin_neon_vaddhn_v: {
1196 llvm::FixedVectorType *SrcTy =
1197 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1198
1199 // %sum = add <4 x i32> %lhs, %rhs
1200 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1201 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: SrcTy);
1202 Ops[0] = Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1], Name: "vaddhn");
1203
1204 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
1205 Constant *ShiftAmt =
1206 ConstantInt::get(Ty: SrcTy, V: SrcTy->getScalarSizeInBits() / 2);
1207 Ops[0] = Builder.CreateLShr(LHS: Ops[0], RHS: ShiftAmt, Name: "vaddhn");
1208
1209 // %res = trunc <4 x i32> %high to <4 x i16>
1210 return Builder.CreateTrunc(V: Ops[0], DestTy: VTy, Name: "vaddhn");
1211 }
1212 case NEON::BI__builtin_neon_vcale_v:
1213 case NEON::BI__builtin_neon_vcaleq_v:
1214 case NEON::BI__builtin_neon_vcalt_v:
1215 case NEON::BI__builtin_neon_vcaltq_v:
1216 std::swap(a&: Ops[0], b&: Ops[1]);
1217 [[fallthrough]];
1218 case NEON::BI__builtin_neon_vcage_v:
1219 case NEON::BI__builtin_neon_vcageq_v:
1220 case NEON::BI__builtin_neon_vcagt_v:
1221 case NEON::BI__builtin_neon_vcagtq_v: {
1222 llvm::Type *Ty;
1223 switch (VTy->getScalarSizeInBits()) {
1224 default: llvm_unreachable("unexpected type");
1225 case 32:
1226 Ty = FloatTy;
1227 break;
1228 case 64:
1229 Ty = DoubleTy;
1230 break;
1231 case 16:
1232 Ty = HalfTy;
1233 break;
1234 }
1235 auto *VecFlt = llvm::FixedVectorType::get(ElementType: Ty, NumElts: VTy->getNumElements());
1236 llvm::Type *Tys[] = { VTy, VecFlt };
1237 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1238 return EmitNeonCall(F, Ops, name: NameHint);
1239 }
1240 case NEON::BI__builtin_neon_vceqz_v:
1241 case NEON::BI__builtin_neon_vceqzq_v:
1242 return EmitAArch64CompareBuiltinExpr(
1243 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OEQ : ICmpInst::ICMP_EQ, Name: "vceqz");
1244 case NEON::BI__builtin_neon_vcgez_v:
1245 case NEON::BI__builtin_neon_vcgezq_v:
1246 return EmitAArch64CompareBuiltinExpr(
1247 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OGE : ICmpInst::ICMP_SGE,
1248 Name: "vcgez");
1249 case NEON::BI__builtin_neon_vclez_v:
1250 case NEON::BI__builtin_neon_vclezq_v:
1251 return EmitAArch64CompareBuiltinExpr(
1252 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OLE : ICmpInst::ICMP_SLE,
1253 Name: "vclez");
1254 case NEON::BI__builtin_neon_vcgtz_v:
1255 case NEON::BI__builtin_neon_vcgtzq_v:
1256 return EmitAArch64CompareBuiltinExpr(
1257 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OGT : ICmpInst::ICMP_SGT,
1258 Name: "vcgtz");
1259 case NEON::BI__builtin_neon_vcltz_v:
1260 case NEON::BI__builtin_neon_vcltzq_v:
1261 return EmitAArch64CompareBuiltinExpr(
1262 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OLT : ICmpInst::ICMP_SLT,
1263 Name: "vcltz");
1264 case NEON::BI__builtin_neon_vclz_v:
1265 case NEON::BI__builtin_neon_vclzq_v:
1266 // We generate target-independent intrinsic, which needs a second argument
1267 // for whether or not clz of zero is undefined; on ARM it isn't.
1268 Ops.push_back(Elt: Builder.getInt1(V: getTarget().isCLZForZeroUndef()));
1269 break;
1270 case NEON::BI__builtin_neon_vcvt_f32_v:
1271 case NEON::BI__builtin_neon_vcvtq_f32_v:
1272 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1273 Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float32, false, Quad),
1274 HasFastHalfType);
1275 return Usgn ? Builder.CreateUIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt")
1276 : Builder.CreateSIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt");
1277 case NEON::BI__builtin_neon_vcvt_f16_s16:
1278 case NEON::BI__builtin_neon_vcvt_f16_u16:
1279 case NEON::BI__builtin_neon_vcvtq_f16_s16:
1280 case NEON::BI__builtin_neon_vcvtq_f16_u16:
1281 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1282 Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float16, false, Quad),
1283 HasFastHalfType);
1284 return Usgn ? Builder.CreateUIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt")
1285 : Builder.CreateSIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt");
1286 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
1287 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
1288 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
1289 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
1290 llvm::Type *Tys[2] = { GetFloatNeonType(CGF: this, IntTypeFlags: Type), Ty };
1291 Function *F = CGM.getIntrinsic(IID: Int, Tys);
1292 return EmitNeonCall(F, Ops, name: "vcvt_n");
1293 }
1294 case NEON::BI__builtin_neon_vcvt_n_f32_v:
1295 case NEON::BI__builtin_neon_vcvt_n_f64_v:
1296 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
1297 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
1298 llvm::Type *Tys[2] = { GetFloatNeonType(CGF: this, IntTypeFlags: Type), Ty };
1299 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
1300 Function *F = CGM.getIntrinsic(IID: Int, Tys);
1301 return EmitNeonCall(F, Ops, name: "vcvt_n");
1302 }
1303 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
1304 case NEON::BI__builtin_neon_vcvt_n_s32_v:
1305 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
1306 case NEON::BI__builtin_neon_vcvt_n_u32_v:
1307 case NEON::BI__builtin_neon_vcvt_n_s64_v:
1308 case NEON::BI__builtin_neon_vcvt_n_u64_v:
1309 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
1310 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
1311 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
1312 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
1313 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
1314 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
1315 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
1316 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1317 return EmitNeonCall(F, Ops, name: "vcvt_n");
1318 }
1319 case NEON::BI__builtin_neon_vcvt_s32_v:
1320 case NEON::BI__builtin_neon_vcvt_u32_v:
1321 case NEON::BI__builtin_neon_vcvt_s64_v:
1322 case NEON::BI__builtin_neon_vcvt_u64_v:
1323 case NEON::BI__builtin_neon_vcvt_s16_f16:
1324 case NEON::BI__builtin_neon_vcvt_u16_f16:
1325 case NEON::BI__builtin_neon_vcvtq_s32_v:
1326 case NEON::BI__builtin_neon_vcvtq_u32_v:
1327 case NEON::BI__builtin_neon_vcvtq_s64_v:
1328 case NEON::BI__builtin_neon_vcvtq_u64_v:
1329 case NEON::BI__builtin_neon_vcvtq_s16_f16:
1330 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
1331 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: GetFloatNeonType(CGF: this, IntTypeFlags: Type));
1332 if (Int) {
1333 // AArch64: use fptosi.sat/fptoui.sat unless under strict FP.
1334 if (!Builder.getIsFPConstrained())
1335 Int = Usgn ? Intrinsic::fptoui_sat : Intrinsic::fptosi_sat;
1336 llvm::Type *Tys[2] = {Ty, Ops[0]->getType()};
1337 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtz");
1338 }
1339 // FIXME: ARM uses plain fptoui/fptosi which have UB on out-of-range
1340 // values. These should also use saturating intrinsics.
1341 return Usgn ? Builder.CreateFPToUI(V: Ops[0], DestTy: Ty, Name: "vcvt")
1342 : Builder.CreateFPToSI(V: Ops[0], DestTy: Ty, Name: "vcvt");
1343 }
1344 case NEON::BI__builtin_neon_vcvta_s16_f16:
1345 case NEON::BI__builtin_neon_vcvta_s32_v:
1346 case NEON::BI__builtin_neon_vcvta_s64_v:
1347 case NEON::BI__builtin_neon_vcvta_u16_f16:
1348 case NEON::BI__builtin_neon_vcvta_u32_v:
1349 case NEON::BI__builtin_neon_vcvta_u64_v:
1350 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
1351 case NEON::BI__builtin_neon_vcvtaq_s32_v:
1352 case NEON::BI__builtin_neon_vcvtaq_s64_v:
1353 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
1354 case NEON::BI__builtin_neon_vcvtaq_u32_v:
1355 case NEON::BI__builtin_neon_vcvtaq_u64_v:
1356 case NEON::BI__builtin_neon_vcvtn_s16_f16:
1357 case NEON::BI__builtin_neon_vcvtn_s32_v:
1358 case NEON::BI__builtin_neon_vcvtn_s64_v:
1359 case NEON::BI__builtin_neon_vcvtn_u16_f16:
1360 case NEON::BI__builtin_neon_vcvtn_u32_v:
1361 case NEON::BI__builtin_neon_vcvtn_u64_v:
1362 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
1363 case NEON::BI__builtin_neon_vcvtnq_s32_v:
1364 case NEON::BI__builtin_neon_vcvtnq_s64_v:
1365 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
1366 case NEON::BI__builtin_neon_vcvtnq_u32_v:
1367 case NEON::BI__builtin_neon_vcvtnq_u64_v:
1368 case NEON::BI__builtin_neon_vcvtp_s16_f16:
1369 case NEON::BI__builtin_neon_vcvtp_s32_v:
1370 case NEON::BI__builtin_neon_vcvtp_s64_v:
1371 case NEON::BI__builtin_neon_vcvtp_u16_f16:
1372 case NEON::BI__builtin_neon_vcvtp_u32_v:
1373 case NEON::BI__builtin_neon_vcvtp_u64_v:
1374 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
1375 case NEON::BI__builtin_neon_vcvtpq_s32_v:
1376 case NEON::BI__builtin_neon_vcvtpq_s64_v:
1377 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
1378 case NEON::BI__builtin_neon_vcvtpq_u32_v:
1379 case NEON::BI__builtin_neon_vcvtpq_u64_v:
1380 case NEON::BI__builtin_neon_vcvtm_s16_f16:
1381 case NEON::BI__builtin_neon_vcvtm_s32_v:
1382 case NEON::BI__builtin_neon_vcvtm_s64_v:
1383 case NEON::BI__builtin_neon_vcvtm_u16_f16:
1384 case NEON::BI__builtin_neon_vcvtm_u32_v:
1385 case NEON::BI__builtin_neon_vcvtm_u64_v:
1386 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
1387 case NEON::BI__builtin_neon_vcvtmq_s32_v:
1388 case NEON::BI__builtin_neon_vcvtmq_s64_v:
1389 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
1390 case NEON::BI__builtin_neon_vcvtmq_u32_v:
1391 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
1392 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
1393 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: NameHint);
1394 }
1395 case NEON::BI__builtin_neon_vcvtx_f32_v: {
1396 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
1397 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: NameHint);
1398
1399 }
1400 case NEON::BI__builtin_neon_vext_v:
1401 case NEON::BI__builtin_neon_vextq_v: {
1402 int CV = cast<ConstantInt>(Val: Ops[2])->getSExtValue();
1403 SmallVector<int, 16> Indices;
1404 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1405 Indices.push_back(Elt: i+CV);
1406
1407 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1408 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1409 return Builder.CreateShuffleVector(V1: Ops[0], V2: Ops[1], Mask: Indices, Name: "vext");
1410 }
1411 case NEON::BI__builtin_neon_vfma_v:
1412 case NEON::BI__builtin_neon_vfmaq_v: {
1413 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1414 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1415 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1416
1417 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
1418 return emitCallMaybeConstrainedFPBuiltin(
1419 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
1420 Args: {Ops[1], Ops[2], Ops[0]});
1421 }
1422 case NEON::BI__builtin_neon_vld1_x2_v:
1423 case NEON::BI__builtin_neon_vld1q_x2_v:
1424 case NEON::BI__builtin_neon_vld1_x3_v:
1425 case NEON::BI__builtin_neon_vld1q_x3_v:
1426 case NEON::BI__builtin_neon_vld1_x4_v:
1427 case NEON::BI__builtin_neon_vld1q_x4_v: {
1428 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
1429 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1430 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld1xN");
1431 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
1432 }
1433 case NEON::BI__builtin_neon_vld1_v:
1434 case NEON::BI__builtin_neon_vld1q_v: {
1435 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1436 Ops.push_back(Elt: getAlignmentValue32(PtrOp0));
1437 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "vld1");
1438 }
1439 case NEON::BI__builtin_neon_vld2_v:
1440 case NEON::BI__builtin_neon_vld2q_v:
1441 case NEON::BI__builtin_neon_vld3_v:
1442 case NEON::BI__builtin_neon_vld3q_v:
1443 case NEON::BI__builtin_neon_vld4_v:
1444 case NEON::BI__builtin_neon_vld4q_v:
1445 case NEON::BI__builtin_neon_vld2_dup_v:
1446 case NEON::BI__builtin_neon_vld2q_dup_v:
1447 case NEON::BI__builtin_neon_vld3_dup_v:
1448 case NEON::BI__builtin_neon_vld3q_dup_v:
1449 case NEON::BI__builtin_neon_vld4_dup_v:
1450 case NEON::BI__builtin_neon_vld4q_dup_v: {
1451 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1452 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1453 Value *Align = getAlignmentValue32(PtrOp1);
1454 Ops[1] = Builder.CreateCall(Callee: F, Args: {Ops[1], Align}, Name: NameHint);
1455 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
1456 }
1457 case NEON::BI__builtin_neon_vld1_dup_v:
1458 case NEON::BI__builtin_neon_vld1q_dup_v: {
1459 Value *V = PoisonValue::get(T: Ty);
1460 PtrOp0 = PtrOp0.withElementType(ElemTy: VTy->getElementType());
1461 LoadInst *Ld = Builder.CreateLoad(Addr: PtrOp0);
1462 llvm::Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
1463 Ops[0] = Builder.CreateInsertElement(Vec: V, NewElt: Ld, Idx: CI);
1464 return EmitNeonSplat(V: Ops[0], C: CI);
1465 }
1466 case NEON::BI__builtin_neon_vld2_lane_v:
1467 case NEON::BI__builtin_neon_vld2q_lane_v:
1468 case NEON::BI__builtin_neon_vld3_lane_v:
1469 case NEON::BI__builtin_neon_vld3q_lane_v:
1470 case NEON::BI__builtin_neon_vld4_lane_v:
1471 case NEON::BI__builtin_neon_vld4q_lane_v: {
1472 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1473 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1474 for (unsigned I = 2; I < Ops.size() - 1; ++I)
1475 Ops[I] = Builder.CreateBitCast(V: Ops[I], DestTy: Ty);
1476 Ops.push_back(Elt: getAlignmentValue32(PtrOp1));
1477 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: NameHint);
1478 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
1479 }
1480 case NEON::BI__builtin_neon_vmovl_v: {
1481 llvm::FixedVectorType *DTy =
1482 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1483 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DTy);
1484 if (Usgn)
1485 return Builder.CreateZExt(V: Ops[0], DestTy: Ty, Name: "vmovl");
1486 return Builder.CreateSExt(V: Ops[0], DestTy: Ty, Name: "vmovl");
1487 }
1488 case NEON::BI__builtin_neon_vmovn_v: {
1489 llvm::FixedVectorType *QTy =
1490 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1491 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: QTy);
1492 return Builder.CreateTrunc(V: Ops[0], DestTy: Ty, Name: "vmovn");
1493 }
1494 case NEON::BI__builtin_neon_vmull_v:
1495 // FIXME: the integer vmull operations could be emitted in terms of pure
1496 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
1497 // hoisting the exts outside loops. Until global ISel comes along that can
1498 // see through such movement this leads to bad CodeGen. So we need an
1499 // intrinsic for now.
1500 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
1501 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
1502 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmull");
1503 case NEON::BI__builtin_neon_vpadal_v:
1504 case NEON::BI__builtin_neon_vpadalq_v: {
1505 // The source operand type has twice as many elements of half the size.
1506 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1507 llvm::Type *EltTy =
1508 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: EltBits / 2);
1509 auto *NarrowTy =
1510 llvm::FixedVectorType::get(ElementType: EltTy, NumElts: VTy->getNumElements() * 2);
1511 llvm::Type *Tys[2] = { Ty, NarrowTy };
1512 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: NameHint);
1513 }
1514 case NEON::BI__builtin_neon_vpaddl_v:
1515 case NEON::BI__builtin_neon_vpaddlq_v: {
1516 // The source operand type has twice as many elements of half the size.
1517 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1518 llvm::Type *EltTy = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: EltBits / 2);
1519 auto *NarrowTy =
1520 llvm::FixedVectorType::get(ElementType: EltTy, NumElts: VTy->getNumElements() * 2);
1521 llvm::Type *Tys[2] = { Ty, NarrowTy };
1522 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vpaddl");
1523 }
1524 case NEON::BI__builtin_neon_vqdmlal_v:
1525 case NEON::BI__builtin_neon_vqdmlsl_v: {
1526 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
1527 Ops[1] =
1528 EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys: Ty), Ops&: MulOps, name: "vqdmlal");
1529 Ops.resize(N: 2);
1530 return EmitNeonCall(F: CGM.getIntrinsic(IID: AltLLVMIntrinsic, Tys: Ty), Ops, name: NameHint);
1531 }
1532 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
1533 case NEON::BI__builtin_neon_vqdmulh_lane_v:
1534 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
1535 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
1536 auto *RTy = cast<llvm::FixedVectorType>(Val: Ty);
1537 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
1538 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
1539 RTy = llvm::FixedVectorType::get(ElementType: RTy->getElementType(),
1540 NumElts: RTy->getNumElements() * 2);
1541 llvm::Type *Tys[2] = {
1542 RTy, GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
1543 /*isQuad*/ false))};
1544 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: NameHint);
1545 }
1546 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
1547 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
1548 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
1549 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
1550 llvm::Type *Tys[2] = {
1551 Ty, GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
1552 /*isQuad*/ true))};
1553 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: NameHint);
1554 }
1555 case NEON::BI__builtin_neon_vqshl_n_v:
1556 case NEON::BI__builtin_neon_vqshlq_n_v:
1557 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshl_n",
1558 shift: 1, rightshift: false);
1559 case NEON::BI__builtin_neon_vqshlu_n_v:
1560 case NEON::BI__builtin_neon_vqshluq_n_v:
1561 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshlu_n",
1562 shift: 1, rightshift: false);
1563 case NEON::BI__builtin_neon_vrecpe_v:
1564 case NEON::BI__builtin_neon_vrecpeq_v:
1565 case NEON::BI__builtin_neon_vrsqrte_v:
1566 case NEON::BI__builtin_neon_vrsqrteq_v:
1567 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
1568 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: NameHint);
1569 case NEON::BI__builtin_neon_vrndi_v:
1570 case NEON::BI__builtin_neon_vrndiq_v:
1571 Int = Builder.getIsFPConstrained()
1572 ? Intrinsic::experimental_constrained_nearbyint
1573 : Intrinsic::nearbyint;
1574 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: NameHint);
1575 case NEON::BI__builtin_neon_vrshr_n_v:
1576 case NEON::BI__builtin_neon_vrshrq_n_v:
1577 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrshr_n",
1578 shift: 1, rightshift: true);
1579 case NEON::BI__builtin_neon_vsha512hq_u64:
1580 case NEON::BI__builtin_neon_vsha512h2q_u64:
1581 case NEON::BI__builtin_neon_vsha512su0q_u64:
1582 case NEON::BI__builtin_neon_vsha512su1q_u64: {
1583 Function *F = CGM.getIntrinsic(IID: Int);
1584 return EmitNeonCall(F, Ops, name: "");
1585 }
1586 case NEON::BI__builtin_neon_vshl_n_v:
1587 case NEON::BI__builtin_neon_vshlq_n_v:
1588 Ops[1] = EmitNeonShiftVector(V: Ops[1], Ty, neg: false);
1589 return Builder.CreateShl(LHS: Builder.CreateBitCast(V: Ops[0],DestTy: Ty), RHS: Ops[1],
1590 Name: "vshl_n");
1591 case NEON::BI__builtin_neon_vshll_n_v: {
1592 llvm::FixedVectorType *SrcTy =
1593 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1594 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1595 if (Usgn)
1596 Ops[0] = Builder.CreateZExt(V: Ops[0], DestTy: VTy);
1597 else
1598 Ops[0] = Builder.CreateSExt(V: Ops[0], DestTy: VTy);
1599 Ops[1] = EmitNeonShiftVector(V: Ops[1], Ty: VTy, neg: false);
1600 return Builder.CreateShl(LHS: Ops[0], RHS: Ops[1], Name: "vshll_n");
1601 }
1602 case NEON::BI__builtin_neon_vshrn_n_v: {
1603 llvm::FixedVectorType *SrcTy =
1604 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1605 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1606 Ops[1] = EmitNeonShiftVector(V: Ops[1], Ty: SrcTy, neg: false);
1607 if (Usgn)
1608 Ops[0] = Builder.CreateLShr(LHS: Ops[0], RHS: Ops[1]);
1609 else
1610 Ops[0] = Builder.CreateAShr(LHS: Ops[0], RHS: Ops[1]);
1611 return Builder.CreateTrunc(V: Ops[0], DestTy: Ty, Name: "vshrn_n");
1612 }
1613 case NEON::BI__builtin_neon_vshr_n_v:
1614 case NEON::BI__builtin_neon_vshrq_n_v:
1615 return EmitNeonRShiftImm(Vec: Ops[0], Shift: Ops[1], Ty, usgn: Usgn, name: "vshr_n");
1616 case NEON::BI__builtin_neon_vst1_v:
1617 case NEON::BI__builtin_neon_vst1q_v:
1618 case NEON::BI__builtin_neon_vst2_v:
1619 case NEON::BI__builtin_neon_vst2q_v:
1620 case NEON::BI__builtin_neon_vst3_v:
1621 case NEON::BI__builtin_neon_vst3q_v:
1622 case NEON::BI__builtin_neon_vst4_v:
1623 case NEON::BI__builtin_neon_vst4q_v:
1624 case NEON::BI__builtin_neon_vst2_lane_v:
1625 case NEON::BI__builtin_neon_vst2q_lane_v:
1626 case NEON::BI__builtin_neon_vst3_lane_v:
1627 case NEON::BI__builtin_neon_vst3q_lane_v:
1628 case NEON::BI__builtin_neon_vst4_lane_v:
1629 case NEON::BI__builtin_neon_vst4q_lane_v: {
1630 llvm::Type *Tys[] = {Int8PtrTy, Ty};
1631 Ops.push_back(Elt: getAlignmentValue32(PtrOp0));
1632 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "");
1633 }
1634 case NEON::BI__builtin_neon_vsm3partw1q_u32:
1635 case NEON::BI__builtin_neon_vsm3partw2q_u32:
1636 case NEON::BI__builtin_neon_vsm3ss1q_u32:
1637 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
1638 case NEON::BI__builtin_neon_vsm4eq_u32: {
1639 Function *F = CGM.getIntrinsic(IID: Int);
1640 return EmitNeonCall(F, Ops, name: "");
1641 }
1642 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
1643 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
1644 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
1645 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
1646 Function *F = CGM.getIntrinsic(IID: Int);
1647 Ops[3] = Builder.CreateZExt(V: Ops[3], DestTy: Int64Ty);
1648 return EmitNeonCall(F, Ops, name: "");
1649 }
1650 case NEON::BI__builtin_neon_vst1_x2_v:
1651 case NEON::BI__builtin_neon_vst1q_x2_v:
1652 case NEON::BI__builtin_neon_vst1_x3_v:
1653 case NEON::BI__builtin_neon_vst1q_x3_v:
1654 case NEON::BI__builtin_neon_vst1_x4_v:
1655 case NEON::BI__builtin_neon_vst1q_x4_v: {
1656 // TODO: Currently in AArch32 mode the pointer operand comes first, whereas
1657 // in AArch64 it comes last. We may want to stick to one or another.
1658 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
1659 Arch == llvm::Triple::aarch64_32) {
1660 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
1661 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
1662 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "");
1663 }
1664 llvm::Type *Tys[2] = {DefaultPtrTy, VTy};
1665 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "");
1666 }
1667 case NEON::BI__builtin_neon_vsubhn_v: {
1668 llvm::FixedVectorType *SrcTy =
1669 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1670
1671 // %sum = add <4 x i32> %lhs, %rhs
1672 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1673 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: SrcTy);
1674 Ops[0] = Builder.CreateSub(LHS: Ops[0], RHS: Ops[1], Name: "vsubhn");
1675
1676 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
1677 Constant *ShiftAmt =
1678 ConstantInt::get(Ty: SrcTy, V: SrcTy->getScalarSizeInBits() / 2);
1679 Ops[0] = Builder.CreateLShr(LHS: Ops[0], RHS: ShiftAmt, Name: "vsubhn");
1680
1681 // %res = trunc <4 x i32> %high to <4 x i16>
1682 return Builder.CreateTrunc(V: Ops[0], DestTy: VTy, Name: "vsubhn");
1683 }
1684 case NEON::BI__builtin_neon_vtrn_v:
1685 case NEON::BI__builtin_neon_vtrnq_v: {
1686 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1687 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1688 Value *SV = nullptr;
1689
1690 for (unsigned vi = 0; vi != 2; ++vi) {
1691 SmallVector<int, 16> Indices;
1692 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1693 Indices.push_back(Elt: i+vi);
1694 Indices.push_back(Elt: i+e+vi);
1695 }
1696 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
1697 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vtrn");
1698 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
1699 }
1700 return SV;
1701 }
1702 case NEON::BI__builtin_neon_vtst_v:
1703 case NEON::BI__builtin_neon_vtstq_v: {
1704 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1705 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1706 Ops[0] = Builder.CreateAnd(LHS: Ops[0], RHS: Ops[1]);
1707 Ops[0] = Builder.CreateICmp(P: ICmpInst::ICMP_NE, LHS: Ops[0],
1708 RHS: ConstantAggregateZero::get(Ty));
1709 return Builder.CreateSExt(V: Ops[0], DestTy: Ty, Name: "vtst");
1710 }
1711 case NEON::BI__builtin_neon_vuzp_v:
1712 case NEON::BI__builtin_neon_vuzpq_v: {
1713 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1714 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1715 Value *SV = nullptr;
1716
1717 for (unsigned vi = 0; vi != 2; ++vi) {
1718 SmallVector<int, 16> Indices;
1719 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1720 Indices.push_back(Elt: 2*i+vi);
1721
1722 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
1723 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vuzp");
1724 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
1725 }
1726 return SV;
1727 }
1728 case NEON::BI__builtin_neon_vxarq_u64: {
1729 Function *F = CGM.getIntrinsic(IID: Int);
1730 Ops[2] = Builder.CreateZExt(V: Ops[2], DestTy: Int64Ty);
1731 return EmitNeonCall(F, Ops, name: "");
1732 }
1733 case NEON::BI__builtin_neon_vzip_v:
1734 case NEON::BI__builtin_neon_vzipq_v: {
1735 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1736 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1737 Value *SV = nullptr;
1738
1739 for (unsigned vi = 0; vi != 2; ++vi) {
1740 SmallVector<int, 16> Indices;
1741 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1742 Indices.push_back(Elt: (i + vi*e) >> 1);
1743 Indices.push_back(Elt: ((i + vi*e) >> 1)+e);
1744 }
1745 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
1746 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vzip");
1747 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
1748 }
1749 return SV;
1750 }
1751 case NEON::BI__builtin_neon_vdot_s32:
1752 case NEON::BI__builtin_neon_vdot_u32:
1753 case NEON::BI__builtin_neon_vdotq_s32:
1754 case NEON::BI__builtin_neon_vdotq_u32: {
1755 auto *InputTy =
1756 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1757 llvm::Type *Tys[2] = { Ty, InputTy };
1758 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vdot");
1759 }
1760 case NEON::BI__builtin_neon_vfmlal_low_f16:
1761 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
1762 auto *InputTy =
1763 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1764 llvm::Type *Tys[2] = { Ty, InputTy };
1765 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlal_low");
1766 }
1767 case NEON::BI__builtin_neon_vfmlsl_low_f16:
1768 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
1769 auto *InputTy =
1770 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1771 llvm::Type *Tys[2] = { Ty, InputTy };
1772 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlsl_low");
1773 }
1774 case NEON::BI__builtin_neon_vfmlal_high_f16:
1775 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
1776 auto *InputTy =
1777 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1778 llvm::Type *Tys[2] = { Ty, InputTy };
1779 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlal_high");
1780 }
1781 case NEON::BI__builtin_neon_vfmlsl_high_f16:
1782 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
1783 auto *InputTy =
1784 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1785 llvm::Type *Tys[2] = { Ty, InputTy };
1786 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlsl_high");
1787 }
1788 case NEON::BI__builtin_neon_vmmlaq_s32:
1789 case NEON::BI__builtin_neon_vmmlaq_u32: {
1790 auto *InputTy =
1791 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1792 llvm::Type *Tys[2] = { Ty, InputTy };
1793 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "vmmla");
1794 }
1795 case NEON::BI__builtin_neon_vmmlaq_f16:
1796 case NEON::BI__builtin_neon_vmmlaq_f32_f16: {
1797 auto *InputTy =
1798 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1799 llvm::Type *Tys[2] = {Ty, InputTy};
1800 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fmmla");
1801 }
1802 case NEON::BI__builtin_neon_vusmmlaq_s32: {
1803 auto *InputTy =
1804 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1805 llvm::Type *Tys[2] = { Ty, InputTy };
1806 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vusmmla");
1807 }
1808 case NEON::BI__builtin_neon_vusdot_s32:
1809 case NEON::BI__builtin_neon_vusdotq_s32: {
1810 auto *InputTy =
1811 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1812 llvm::Type *Tys[2] = { Ty, InputTy };
1813 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vusdot");
1814 }
1815 case NEON::BI__builtin_neon_vbfdot_f32:
1816 case NEON::BI__builtin_neon_vbfdotq_f32: {
1817 llvm::Type *InputTy =
1818 llvm::FixedVectorType::get(ElementType: BFloatTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1819 llvm::Type *Tys[2] = { Ty, InputTy };
1820 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vbfdot");
1821 }
1822 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
1823 llvm::Type *Tys[1] = { Ty };
1824 Function *F = CGM.getIntrinsic(IID: Int, Tys);
1825 return EmitNeonCall(F, Ops, name: "vcvtfp2bf");
1826 }
1827
1828 }
1829
1830 assert(Int && "Expected valid intrinsic number");
1831
1832 // Determine the type(s) of this overloaded AArch64 intrinsic.
1833 Function *F = LookupNeonLLVMIntrinsic(IntrinsicID: Int, Modifier, ArgType: Ty, E);
1834
1835 Value *Result = EmitNeonCall(F, Ops, name: NameHint);
1836 llvm::Type *ResultType = ConvertType(T: E->getType());
1837 // AArch64 intrinsic one-element vector type cast to
1838 // scalar type expected by the builtin
1839 return Builder.CreateBitCast(V: Result, DestTy: ResultType, Name: NameHint);
1840}
1841
1842Value *
1843CodeGenFunction::EmitAArch64CompareBuiltinExpr(Value *Op, llvm::Type *Ty,
1844 const CmpInst::Predicate Pred,
1845 const Twine &Name) {
1846
1847 if (isa<FixedVectorType>(Val: Ty)) {
1848 // Vector types are cast to i8 vectors. Recover original type.
1849 Op = Builder.CreateBitCast(V: Op, DestTy: Ty);
1850 }
1851
1852 Constant *zero = Constant::getNullValue(Ty: Op->getType());
1853
1854 if (CmpInst::isFPPredicate(P: Pred)) {
1855 if (Pred == CmpInst::FCMP_OEQ)
1856 Op = Builder.CreateFCmp(P: Pred, LHS: Op, RHS: zero);
1857 else
1858 Op = Builder.CreateFCmpS(P: Pred, LHS: Op, RHS: zero);
1859 } else {
1860 Op = Builder.CreateICmp(P: Pred, LHS: Op, RHS: zero);
1861 }
1862
1863 llvm::Type *ResTy = Ty;
1864 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty))
1865 ResTy = FixedVectorType::get(
1866 ElementType: IntegerType::get(C&: getLLVMContext(), NumBits: VTy->getScalarSizeInBits()),
1867 NumElts: VTy->getNumElements());
1868
1869 return Builder.CreateSExt(V: Op, DestTy: ResTy, Name);
1870}
1871
1872static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
1873 Value *ExtOp, Value *IndexOp,
1874 llvm::Type *ResTy, unsigned IntID,
1875 const char *Name) {
1876 SmallVector<Value *, 2> TblOps;
1877 if (ExtOp)
1878 TblOps.push_back(Elt: ExtOp);
1879
1880 // Build a vector containing sequential number like (0, 1, 2, ..., 15)
1881 SmallVector<int, 16> Indices;
1882 auto *TblTy = cast<llvm::FixedVectorType>(Val: Ops[0]->getType());
1883 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
1884 Indices.push_back(Elt: 2*i);
1885 Indices.push_back(Elt: 2*i+1);
1886 }
1887
1888 int PairPos = 0, End = Ops.size() - 1;
1889 while (PairPos < End) {
1890 TblOps.push_back(Elt: CGF.Builder.CreateShuffleVector(V1: Ops[PairPos],
1891 V2: Ops[PairPos+1], Mask: Indices,
1892 Name));
1893 PairPos += 2;
1894 }
1895
1896 // If there's an odd number of 64-bit lookup table, fill the high 64-bit
1897 // of the 128-bit lookup table with zero.
1898 if (PairPos == End) {
1899 Value *ZeroTbl = ConstantAggregateZero::get(Ty: TblTy);
1900 TblOps.push_back(Elt: CGF.Builder.CreateShuffleVector(V1: Ops[PairPos],
1901 V2: ZeroTbl, Mask: Indices, Name));
1902 }
1903
1904 Function *TblF;
1905 TblOps.push_back(Elt: IndexOp);
1906 TblF = CGF.CGM.getIntrinsic(IID: IntID, Tys: ResTy);
1907
1908 return CGF.EmitNeonCall(F: TblF, Ops&: TblOps, name: Name);
1909}
1910
1911Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
1912 unsigned Value;
1913 switch (BuiltinID) {
1914 default:
1915 return nullptr;
1916 case clang::ARM::BI__builtin_arm_nop:
1917 Value = 0;
1918 break;
1919 case clang::ARM::BI__builtin_arm_yield:
1920 case clang::ARM::BI__yield:
1921 Value = 1;
1922 break;
1923 case clang::ARM::BI__builtin_arm_wfe:
1924 case clang::ARM::BI__wfe:
1925 Value = 2;
1926 break;
1927 case clang::ARM::BI__builtin_arm_wfi:
1928 case clang::ARM::BI__wfi:
1929 Value = 3;
1930 break;
1931 case clang::ARM::BI__builtin_arm_sev:
1932 case clang::ARM::BI__sev:
1933 Value = 4;
1934 break;
1935 case clang::ARM::BI__builtin_arm_sevl:
1936 case clang::ARM::BI__sevl:
1937 Value = 5;
1938 break;
1939 }
1940
1941 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_hint),
1942 Args: llvm::ConstantInt::get(Ty: Int32Ty, V: Value));
1943}
1944
1945enum SpecialRegisterAccessKind {
1946 NormalRead,
1947 VolatileRead,
1948 Write,
1949};
1950
1951// Generates the IR for the read/write special register builtin,
1952// ValueType is the type of the value that is to be written or read,
1953// RegisterType is the type of the register being written to or read from.
1954static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF,
1955 const CallExpr *E,
1956 llvm::Type *RegisterType,
1957 llvm::Type *ValueType,
1958 SpecialRegisterAccessKind AccessKind,
1959 StringRef SysReg = "") {
1960 // write and register intrinsics only support 32, 64 and 128 bit operations.
1961 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
1962 RegisterType->isIntegerTy(128)) &&
1963 "Unsupported size for register.");
1964
1965 CodeGen::CGBuilderTy &Builder = CGF.Builder;
1966 CodeGen::CodeGenModule &CGM = CGF.CGM;
1967 LLVMContext &Context = CGM.getLLVMContext();
1968
1969 if (SysReg.empty()) {
1970 const Expr *SysRegStrExpr = E->getArg(Arg: 0)->IgnoreParenCasts();
1971 SysReg = cast<clang::StringLiteral>(Val: SysRegStrExpr)->getString();
1972 }
1973
1974 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, Str: SysReg) };
1975 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
1976 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
1977
1978 llvm::Type *Types[] = { RegisterType };
1979
1980 bool MixedTypes = RegisterType->isIntegerTy(BitWidth: 64) && ValueType->isIntegerTy(BitWidth: 32);
1981 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
1982 && "Can't fit 64-bit value in 32-bit register");
1983
1984 if (AccessKind != Write) {
1985 assert(AccessKind == NormalRead || AccessKind == VolatileRead);
1986 llvm::Function *F = CGM.getIntrinsic(
1987 IID: AccessKind == VolatileRead ? Intrinsic::read_volatile_register
1988 : Intrinsic::read_register,
1989 Tys: Types);
1990 llvm::Value *Call = Builder.CreateCall(Callee: F, Args: Metadata);
1991
1992 if (MixedTypes)
1993 // Read into 64 bit register and then truncate result to 32 bit.
1994 return Builder.CreateTrunc(V: Call, DestTy: ValueType);
1995
1996 if (ValueType->isPointerTy())
1997 // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*).
1998 return Builder.CreateIntToPtr(V: Call, DestTy: ValueType);
1999
2000 return Call;
2001 }
2002
2003 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::write_register, Tys: Types);
2004 llvm::Value *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
2005 if (MixedTypes) {
2006 // Extend 32 bit write value to 64 bit to pass to write.
2007 ArgValue = Builder.CreateZExt(V: ArgValue, DestTy: RegisterType);
2008 return Builder.CreateCall(Callee: F, Args: { Metadata, ArgValue });
2009 }
2010
2011 if (ValueType->isPointerTy()) {
2012 // Have VoidPtrTy ArgValue but want to return an i32/i64.
2013 ArgValue = Builder.CreatePtrToInt(V: ArgValue, DestTy: RegisterType);
2014 return Builder.CreateCall(Callee: F, Args: { Metadata, ArgValue });
2015 }
2016
2017 return Builder.CreateCall(Callee: F, Args: { Metadata, ArgValue });
2018}
2019
2020static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
2021 const CallExpr *E) {
2022 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2023 CodeGen::CodeGenModule &CGM = CGF.CGM;
2024 SmallVector<llvm::Value *, 4> Ops;
2025
2026 auto getIntArg = [&](unsigned ArgNo) {
2027 Expr::EvalResult Result;
2028 if (!E->getArg(Arg: ArgNo)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
2029 llvm_unreachable("Expected constant argument to range prefetch.");
2030 return Result.Val.getInt().getExtValue();
2031 };
2032
2033 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 0))); /*Addr*/
2034 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 1))); /*Access Kind*/
2035 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 2))); /*Policy*/
2036
2037 if (BuiltinID == clang::AArch64::BI__builtin_arm_range_prefetch_x) {
2038 auto Length = getIntArg(3);
2039 auto Count = getIntArg(4) - 1;
2040 auto Stride = getIntArg(5);
2041 auto Distance = getIntArg(6);
2042
2043 // Map ReuseDistance given in bytes to four bits representing decreasing
2044 // powers of two in the range 512MiB (0b0001) to 32KiB (0b1111). Values
2045 // are rounded up to the nearest power of 2, starting at 32KiB. Any value
2046 // over the maximum is represented by 0 (distance not known).
2047 if (Distance > 0) {
2048 Distance = llvm::Log2_32_Ceil(Value: Distance);
2049 if (Distance < 15)
2050 Distance = 15;
2051 else if (Distance > 29)
2052 Distance = 0;
2053 else
2054 Distance = 30 - Distance;
2055 }
2056
2057 uint64_t Mask22 = (1ULL << 22) - 1;
2058 uint64_t Mask16 = (1ULL << 16) - 1;
2059 uint64_t Metadata = (Distance << 60) | ((Stride & Mask22) << 38) |
2060 ((Count & Mask16) << 22) | (Length & Mask22);
2061
2062 Ops.push_back(Elt: llvm::ConstantInt::get(Ty: Builder.getInt64Ty(), V: Metadata));
2063 } else
2064 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 3)));
2065
2066 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_range_prefetch),
2067 Args: Ops);
2068}
2069
2070static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E,
2071 Instruction *Atomic, unsigned PtrOperand) {
2072 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2073 LLVMContext &Ctx = CGF.CGM.getLLVMContext();
2074 Expr::EvalResult Result;
2075 if (!E->getArg(Arg: 3)->EvaluateAsInt(Result, Ctx: CGF.getContext()))
2076 llvm_unreachable(
2077 "Expected integer hint argument to atomic operation with hint.");
2078 unsigned HintArg = Result.Val.getInt().getExtValue();
2079
2080 auto MemoryHint = toAArch64MemoryHint(I: HintArg);
2081 if (MemoryHint == AArch64MemoryHint::NONE ||
2082 (isa<AtomicRMWInst>(Val: Atomic) && MemoryHint != AArch64MemoryHint::SHUH &&
2083 MemoryHint != AArch64MemoryHint::SHUH_PH))
2084 return;
2085
2086 MDNode *MemHint = MDNode::get(
2087 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "aarch64.mem_hint"),
2088 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: HintArg))});
2089 MDNode *HintNode = MDNode::get(
2090 Context&: Ctx,
2091 MDs: {llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: PtrOperand)), MemHint});
2092 Atomic->setMetadata(KindID: llvm::LLVMContext::MD_mem_cache_hint, Node: HintNode);
2093}
2094
2095static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
2096 unsigned BuiltinID,
2097 const CallExpr *E) {
2098 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2099 CodeGen::CodeGenModule &CGM = CGF.CGM;
2100 Expr::EvalResult Result;
2101 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
2102 llvm_unreachable(
2103 "Expected integer policy argument to atomic store with hint.");
2104
2105 const Expr *Ptr = E->getArg(Arg: 0);
2106 Address Addr = CGF.EmitPointerWithAlignment(Addr: Ptr);
2107 Addr = Addr.withElementType(
2108 ElemTy: CGF.ConvertTypeForMem(T: Ptr->getType()->getPointeeType()));
2109
2110 const Expr *Data = E->getArg(Arg: 1);
2111 Value *DataVal = CGF.EmitToMemory(Value: CGF.EmitScalarExpr(E: Data), Ty: Data->getType());
2112
2113 StoreInst *Store = Builder.CreateStore(Val: DataVal, Addr);
2114 Store->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
2115
2116 AtomicOrdering Ordering;
2117 unsigned OrderingArg = Result.Val.getInt().getExtValue();
2118 assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
2119
2120 switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
2121 default:
2122 llvm_unreachable("Unsupported atomic ordering found.");
2123 case AtomicOrderingCABI::relaxed:
2124 Ordering = AtomicOrdering::Monotonic;
2125 break;
2126 case AtomicOrderingCABI::release:
2127 Ordering = AtomicOrdering::Release;
2128 break;
2129 case AtomicOrderingCABI::seq_cst:
2130 Ordering = AtomicOrdering::SequentiallyConsistent;
2131 break;
2132 }
2133 Store->setAtomic(Ordering);
2134 AttachAtomicHintMetadata(CGF, E, Atomic: Store, PtrOperand: 1);
2135 return Store;
2136}
2137
2138static Value *EmitAtomicFetchWithHintBuiltin(CodeGenFunction &CGF,
2139 const CallExpr *E,
2140 AtomicRMWInst::BinOp Op) {
2141 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2142 Expr::EvalResult Result;
2143 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result, Ctx: CGF.getContext()))
2144 llvm_unreachable(
2145 "Expected integer policy argument to atomic fetch with hint.");
2146
2147 AtomicOrdering Ordering;
2148 unsigned OrderingArg = Result.Val.getInt().getExtValue();
2149 assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
2150
2151 switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
2152 case AtomicOrderingCABI::relaxed:
2153 Ordering = AtomicOrdering::Monotonic;
2154 break;
2155 case AtomicOrderingCABI::consume:
2156 case AtomicOrderingCABI::acquire:
2157 Ordering = AtomicOrdering::Acquire;
2158 break;
2159 case AtomicOrderingCABI::release:
2160 Ordering = AtomicOrdering::Release;
2161 break;
2162 case AtomicOrderingCABI::acq_rel:
2163 Ordering = AtomicOrdering::AcquireRelease;
2164 break;
2165 case AtomicOrderingCABI::seq_cst:
2166 Ordering = AtomicOrdering::SequentiallyConsistent;
2167 break;
2168 }
2169
2170 const Expr *Ptr = E->getArg(Arg: 0);
2171 Address Addr = CGF.EmitPointerWithAlignment(Addr: Ptr);
2172 Addr = Addr.withElementType(
2173 ElemTy: CGF.ConvertTypeForMem(T: Ptr->getType()->getPointeeType()));
2174 const Expr *Data = E->getArg(Arg: 1);
2175 Value *DataVal = CGF.EmitToMemory(Value: CGF.EmitScalarExpr(E: Data), Ty: Data->getType());
2176
2177 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(Op, Addr, Val: DataVal, Ordering);
2178 RMW->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
2179 AttachAtomicHintMetadata(CGF, E, Atomic: RMW, PtrOperand: 0);
2180 return CGF.EmitFromMemory(Value: RMW, Ty: E->getType());
2181}
2182
2183Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
2184 const CallExpr *E,
2185 ReturnValueSlot ReturnValue,
2186 llvm::Triple::ArchType Arch) {
2187 if (auto Hint = GetValueForARMHint(BuiltinID))
2188 return Hint;
2189
2190 if (BuiltinID == clang::ARM::BI__emit) {
2191 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
2192 llvm::FunctionType *FTy =
2193 llvm::FunctionType::get(Result: VoidTy, /*Variadic=*/isVarArg: false);
2194
2195 Expr::EvalResult Result;
2196 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
2197 llvm_unreachable("Sema will ensure that the parameter is constant");
2198
2199 llvm::APSInt Value = Result.Val.getInt();
2200 uint64_t ZExtValue = Value.zextOrTrunc(width: IsThumb ? 16 : 32).getZExtValue();
2201
2202 llvm::InlineAsm *Emit =
2203 IsThumb ? InlineAsm::get(Ty: FTy, AsmString: ".inst.n 0x" + utohexstr(X: ZExtValue), Constraints: "",
2204 /*hasSideEffects=*/true)
2205 : InlineAsm::get(Ty: FTy, AsmString: ".inst 0x" + utohexstr(X: ZExtValue), Constraints: "",
2206 /*hasSideEffects=*/true);
2207
2208 return Builder.CreateCall(Callee: Emit);
2209 }
2210
2211 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
2212 Value *Option = EmitScalarExpr(E: E->getArg(Arg: 0));
2213 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_dbg), Args: Option);
2214 }
2215
2216 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
2217 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
2218 Value *RW = EmitScalarExpr(E: E->getArg(Arg: 1));
2219 Value *IsData = EmitScalarExpr(E: E->getArg(Arg: 2));
2220
2221 // Locality is not supported on ARM target
2222 Value *Locality = llvm::ConstantInt::get(Ty: Int32Ty, V: 3);
2223
2224 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
2225 return Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, IsData});
2226 }
2227
2228 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
2229 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2230 return Builder.CreateCall(
2231 Callee: CGM.getIntrinsic(IID: Intrinsic::bitreverse, Tys: Arg->getType()), Args: Arg, Name: "rbit");
2232 }
2233
2234 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
2235 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
2236 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2237 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: Arg->getType());
2238 Value *Res = Builder.CreateCall(Callee: F, Args: {Arg, Builder.getInt1(V: false)});
2239 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
2240 Res = Builder.CreateTrunc(V: Res, DestTy: Builder.getInt32Ty());
2241 return Res;
2242 }
2243
2244
2245 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
2246 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2247 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_cls), Args: Arg, Name: "cls");
2248 }
2249 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
2250 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2251 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_cls64), Args: Arg,
2252 Name: "cls");
2253 }
2254
2255 if (BuiltinID == clang::ARM::BI__clear_cache) {
2256 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
2257 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
2258 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache, Tys: {CGM.DefaultPtrTy});
2259 return Builder.CreateCall(Callee: F, Args: {Begin, End});
2260 }
2261
2262 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
2263 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
2264 Function *F;
2265
2266 switch (BuiltinID) {
2267 default: llvm_unreachable("unexpected builtin");
2268 case clang::ARM::BI__builtin_arm_mcrr:
2269 F = CGM.getIntrinsic(IID: Intrinsic::arm_mcrr);
2270 break;
2271 case clang::ARM::BI__builtin_arm_mcrr2:
2272 F = CGM.getIntrinsic(IID: Intrinsic::arm_mcrr2);
2273 break;
2274 }
2275
2276 // MCRR{2} instruction has 5 operands but
2277 // the intrinsic has 4 because Rt and Rt2
2278 // are represented as a single unsigned 64
2279 // bit integer in the intrinsic definition
2280 // but internally it's represented as 2 32
2281 // bit integers.
2282
2283 Value *Coproc = EmitScalarExpr(E: E->getArg(Arg: 0));
2284 Value *Opc1 = EmitScalarExpr(E: E->getArg(Arg: 1));
2285 Value *RtAndRt2 = EmitScalarExpr(E: E->getArg(Arg: 2));
2286 Value *CRm = EmitScalarExpr(E: E->getArg(Arg: 3));
2287
2288 Value *C1 = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2289 Value *Rt = Builder.CreateTruncOrBitCast(V: RtAndRt2, DestTy: Int32Ty);
2290 Value *Rt2 = Builder.CreateLShr(LHS: RtAndRt2, RHS: C1);
2291 Rt2 = Builder.CreateTruncOrBitCast(V: Rt2, DestTy: Int32Ty);
2292
2293 return Builder.CreateCall(Callee: F, Args: {Coproc, Opc1, Rt, Rt2, CRm});
2294 }
2295
2296 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
2297 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
2298 Function *F;
2299
2300 switch (BuiltinID) {
2301 default: llvm_unreachable("unexpected builtin");
2302 case clang::ARM::BI__builtin_arm_mrrc:
2303 F = CGM.getIntrinsic(IID: Intrinsic::arm_mrrc);
2304 break;
2305 case clang::ARM::BI__builtin_arm_mrrc2:
2306 F = CGM.getIntrinsic(IID: Intrinsic::arm_mrrc2);
2307 break;
2308 }
2309
2310 Value *Coproc = EmitScalarExpr(E: E->getArg(Arg: 0));
2311 Value *Opc1 = EmitScalarExpr(E: E->getArg(Arg: 1));
2312 Value *CRm = EmitScalarExpr(E: E->getArg(Arg: 2));
2313 Value *RtAndRt2 = Builder.CreateCall(Callee: F, Args: {Coproc, Opc1, CRm});
2314
2315 // Returns an unsigned 64 bit integer, represented
2316 // as two 32 bit integers.
2317
2318 Value *Rt = Builder.CreateExtractValue(Agg: RtAndRt2, Idxs: 1);
2319 Value *Rt1 = Builder.CreateExtractValue(Agg: RtAndRt2, Idxs: 0);
2320 Rt = Builder.CreateZExt(V: Rt, DestTy: Int64Ty);
2321 Rt1 = Builder.CreateZExt(V: Rt1, DestTy: Int64Ty);
2322
2323 Value *ShiftCast = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2324 RtAndRt2 = Builder.CreateShl(LHS: Rt, RHS: ShiftCast, Name: "shl", HasNUW: true);
2325 RtAndRt2 = Builder.CreateOr(LHS: RtAndRt2, RHS: Rt1);
2326
2327 return Builder.CreateBitCast(V: RtAndRt2, DestTy: ConvertType(T: E->getType()));
2328 }
2329
2330 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
2331 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2332 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
2333 getContext().getTypeSize(T: E->getType()) == 64) ||
2334 BuiltinID == clang::ARM::BI__ldrexd) {
2335 Function *F;
2336
2337 switch (BuiltinID) {
2338 default: llvm_unreachable("unexpected builtin");
2339 case clang::ARM::BI__builtin_arm_ldaex:
2340 F = CGM.getIntrinsic(IID: Intrinsic::arm_ldaexd);
2341 break;
2342 case clang::ARM::BI__builtin_arm_ldrexd:
2343 case clang::ARM::BI__builtin_arm_ldrex:
2344 case clang::ARM::BI__ldrexd:
2345 F = CGM.getIntrinsic(IID: Intrinsic::arm_ldrexd);
2346 break;
2347 }
2348
2349 Value *LdPtr = EmitScalarExpr(E: E->getArg(Arg: 0));
2350 Value *Val = Builder.CreateCall(Callee: F, Args: LdPtr, Name: "ldrexd");
2351
2352 Value *Val0 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
2353 Value *Val1 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
2354 Val0 = Builder.CreateZExt(V: Val0, DestTy: Int64Ty);
2355 Val1 = Builder.CreateZExt(V: Val1, DestTy: Int64Ty);
2356
2357 Value *ShiftCst = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2358 Val = Builder.CreateShl(LHS: Val0, RHS: ShiftCst, Name: "shl", HasNUW: true /* nuw */);
2359 Val = Builder.CreateOr(LHS: Val, RHS: Val1);
2360 return Builder.CreateBitCast(V: Val, DestTy: ConvertType(T: E->getType()));
2361 }
2362
2363 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2364 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
2365 Value *LoadAddr = EmitScalarExpr(E: E->getArg(Arg: 0));
2366
2367 QualType Ty = E->getType();
2368 llvm::Type *RealResTy = ConvertType(T: Ty);
2369 llvm::Type *IntTy =
2370 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
2371
2372 Function *F = CGM.getIntrinsic(
2373 IID: BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
2374 : Intrinsic::arm_ldrex,
2375 Tys: DefaultPtrTy);
2376 CallInst *Val = Builder.CreateCall(Callee: F, Args: LoadAddr, Name: "ldrex");
2377 Val->addParamAttr(
2378 ArgNo: 0, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: IntTy));
2379
2380 if (RealResTy->isPointerTy())
2381 return Builder.CreateIntToPtr(V: Val, DestTy: RealResTy);
2382 else {
2383 llvm::Type *IntResTy = llvm::IntegerType::get(
2384 C&: getLLVMContext(), NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: RealResTy));
2385 return Builder.CreateBitCast(V: Builder.CreateTruncOrBitCast(V: Val, DestTy: IntResTy),
2386 DestTy: RealResTy);
2387 }
2388 }
2389
2390 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
2391 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
2392 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
2393 getContext().getTypeSize(T: E->getArg(Arg: 0)->getType()) == 64)) {
2394 Function *F = CGM.getIntrinsic(
2395 IID: BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
2396 : Intrinsic::arm_strexd);
2397 llvm::Type *STy = llvm::StructType::get(elt1: Int32Ty, elts: Int32Ty);
2398
2399 Address Tmp = CreateMemTempWithoutCast(T: E->getArg(Arg: 0)->getType());
2400 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
2401 Builder.CreateStore(Val, Addr: Tmp);
2402
2403 Address LdPtr = Tmp.withElementType(ElemTy: STy);
2404 Val = Builder.CreateLoad(Addr: LdPtr);
2405
2406 Value *Arg0 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
2407 Value *Arg1 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
2408 Value *StPtr = EmitScalarExpr(E: E->getArg(Arg: 1));
2409 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1, StPtr}, Name: "strexd");
2410 }
2411
2412 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
2413 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
2414 Value *StoreVal = EmitScalarExpr(E: E->getArg(Arg: 0));
2415 Value *StoreAddr = EmitScalarExpr(E: E->getArg(Arg: 1));
2416
2417 QualType Ty = E->getArg(Arg: 0)->getType();
2418 llvm::Type *StoreTy =
2419 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
2420
2421 if (StoreVal->getType()->isPointerTy())
2422 StoreVal = Builder.CreatePtrToInt(V: StoreVal, DestTy: Int32Ty);
2423 else {
2424 llvm::Type *IntTy = llvm::IntegerType::get(
2425 C&: getLLVMContext(),
2426 NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: StoreVal->getType()));
2427 StoreVal = Builder.CreateBitCast(V: StoreVal, DestTy: IntTy);
2428 StoreVal = Builder.CreateZExtOrBitCast(V: StoreVal, DestTy: Int32Ty);
2429 }
2430
2431 Function *F = CGM.getIntrinsic(
2432 IID: BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
2433 : Intrinsic::arm_strex,
2434 Tys: StoreAddr->getType());
2435
2436 CallInst *CI = Builder.CreateCall(Callee: F, Args: {StoreVal, StoreAddr}, Name: "strex");
2437 CI->addParamAttr(
2438 ArgNo: 1, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: StoreTy));
2439 return CI;
2440 }
2441
2442 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
2443 Function *F = CGM.getIntrinsic(IID: Intrinsic::arm_clrex);
2444 return Builder.CreateCall(Callee: F);
2445 }
2446
2447 // CRC32
2448 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
2449 switch (BuiltinID) {
2450 case clang::ARM::BI__builtin_arm_crc32b:
2451 CRCIntrinsicID = Intrinsic::arm_crc32b; break;
2452 case clang::ARM::BI__builtin_arm_crc32cb:
2453 CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
2454 case clang::ARM::BI__builtin_arm_crc32h:
2455 CRCIntrinsicID = Intrinsic::arm_crc32h; break;
2456 case clang::ARM::BI__builtin_arm_crc32ch:
2457 CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
2458 case clang::ARM::BI__builtin_arm_crc32w:
2459 case clang::ARM::BI__builtin_arm_crc32d:
2460 CRCIntrinsicID = Intrinsic::arm_crc32w; break;
2461 case clang::ARM::BI__builtin_arm_crc32cw:
2462 case clang::ARM::BI__builtin_arm_crc32cd:
2463 CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
2464 }
2465
2466 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
2467 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
2468 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
2469
2470 // crc32{c,}d intrinsics are implemented as two calls to crc32{c,}w
2471 // intrinsics, hence we need different codegen for these cases.
2472 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
2473 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
2474 Value *C1 = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2475 Value *Arg1a = Builder.CreateTruncOrBitCast(V: Arg1, DestTy: Int32Ty);
2476 Value *Arg1b = Builder.CreateLShr(LHS: Arg1, RHS: C1);
2477 Arg1b = Builder.CreateTruncOrBitCast(V: Arg1b, DestTy: Int32Ty);
2478
2479 Function *F = CGM.getIntrinsic(IID: CRCIntrinsicID);
2480 Value *Res = Builder.CreateCall(Callee: F, Args: {Arg0, Arg1a});
2481 return Builder.CreateCall(Callee: F, Args: {Res, Arg1b});
2482 } else {
2483 Arg1 = Builder.CreateZExtOrBitCast(V: Arg1, DestTy: Int32Ty);
2484
2485 Function *F = CGM.getIntrinsic(IID: CRCIntrinsicID);
2486 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1});
2487 }
2488 }
2489
2490 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2491 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2492 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2493 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
2494 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
2495 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
2496
2497 SpecialRegisterAccessKind AccessKind = Write;
2498 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2499 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2500 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
2501 AccessKind = VolatileRead;
2502
2503 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2504 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
2505
2506 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2507 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
2508
2509 llvm::Type *ValueType;
2510 llvm::Type *RegisterType;
2511 if (IsPointerBuiltin) {
2512 ValueType = VoidPtrTy;
2513 RegisterType = Int32Ty;
2514 } else if (Is64Bit) {
2515 ValueType = RegisterType = Int64Ty;
2516 } else {
2517 ValueType = RegisterType = Int32Ty;
2518 }
2519
2520 return EmitSpecialRegisterBuiltin(CGF&: *this, E, RegisterType, ValueType,
2521 AccessKind);
2522 }
2523
2524 if (BuiltinID == ARM::BI__builtin_sponentry) {
2525 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: AllocaInt8PtrTy);
2526 return Builder.CreateCall(Callee: F);
2527 }
2528
2529 // Handle MSVC intrinsics before argument evaluation to prevent double
2530 // evaluation.
2531 if (std::optional<MSVCIntrin> MsvcIntId = translateArmToMsvcIntrin(BuiltinID))
2532 return EmitMSVCBuiltinExpr(BuiltinID: *MsvcIntId, E);
2533
2534 // Deal with MVE builtins
2535 if (Value *Result = EmitARMMVEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
2536 return Result;
2537 // Handle CDE builtins
2538 if (Value *Result = EmitARMCDEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
2539 return Result;
2540
2541 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
2542 auto It = llvm::find_if(Range: NEONEquivalentIntrinsicMap, P: [BuiltinID](auto &P) {
2543 return P.first == BuiltinID;
2544 });
2545 if (It != end(arr: NEONEquivalentIntrinsicMap))
2546 BuiltinID = It->second;
2547
2548 // Find out if any arguments are required to be integer constant
2549 // expressions.
2550 unsigned ICEArguments = 0;
2551 ASTContext::GetBuiltinTypeError Error;
2552 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
2553 assert(Error == ASTContext::GE_None && "Should not codegen an error");
2554
2555 auto getAlignmentValue32 = [&](Address addr) -> Value* {
2556 return Builder.getInt32(C: addr.getAlignment().getQuantity());
2557 };
2558
2559 Address PtrOp0 = Address::invalid();
2560 Address PtrOp1 = Address::invalid();
2561 SmallVector<Value*, 4> Ops;
2562 bool HasExtraArg = CodeGenUtils::hasExtraNeonArgument(BuiltinID);
2563 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
2564 for (unsigned i = 0, e = NumArgs; i != e; i++) {
2565 if (i == 0) {
2566 switch (BuiltinID) {
2567 case NEON::BI__builtin_neon_vld1_v:
2568 case NEON::BI__builtin_neon_vld1q_v:
2569 case NEON::BI__builtin_neon_vld1q_lane_v:
2570 case NEON::BI__builtin_neon_vld1_lane_v:
2571 case NEON::BI__builtin_neon_vld1_dup_v:
2572 case NEON::BI__builtin_neon_vld1q_dup_v:
2573 case NEON::BI__builtin_neon_vst1_v:
2574 case NEON::BI__builtin_neon_vst1q_v:
2575 case NEON::BI__builtin_neon_vst1q_lane_v:
2576 case NEON::BI__builtin_neon_vst1_lane_v:
2577 case NEON::BI__builtin_neon_vst2_v:
2578 case NEON::BI__builtin_neon_vst2q_v:
2579 case NEON::BI__builtin_neon_vst2_lane_v:
2580 case NEON::BI__builtin_neon_vst2q_lane_v:
2581 case NEON::BI__builtin_neon_vst3_v:
2582 case NEON::BI__builtin_neon_vst3q_v:
2583 case NEON::BI__builtin_neon_vst3_lane_v:
2584 case NEON::BI__builtin_neon_vst3q_lane_v:
2585 case NEON::BI__builtin_neon_vst4_v:
2586 case NEON::BI__builtin_neon_vst4q_v:
2587 case NEON::BI__builtin_neon_vst4_lane_v:
2588 case NEON::BI__builtin_neon_vst4q_lane_v:
2589 // Get the alignment for the argument in addition to the value;
2590 // we'll use it later.
2591 PtrOp0 = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
2592 Ops.push_back(Elt: PtrOp0.emitRawPointer(CGF&: *this));
2593 continue;
2594 }
2595 }
2596 if (i == 1) {
2597 switch (BuiltinID) {
2598 case NEON::BI__builtin_neon_vld2_v:
2599 case NEON::BI__builtin_neon_vld2q_v:
2600 case NEON::BI__builtin_neon_vld3_v:
2601 case NEON::BI__builtin_neon_vld3q_v:
2602 case NEON::BI__builtin_neon_vld4_v:
2603 case NEON::BI__builtin_neon_vld4q_v:
2604 case NEON::BI__builtin_neon_vld2_lane_v:
2605 case NEON::BI__builtin_neon_vld2q_lane_v:
2606 case NEON::BI__builtin_neon_vld3_lane_v:
2607 case NEON::BI__builtin_neon_vld3q_lane_v:
2608 case NEON::BI__builtin_neon_vld4_lane_v:
2609 case NEON::BI__builtin_neon_vld4q_lane_v:
2610 case NEON::BI__builtin_neon_vld2_dup_v:
2611 case NEON::BI__builtin_neon_vld2q_dup_v:
2612 case NEON::BI__builtin_neon_vld3_dup_v:
2613 case NEON::BI__builtin_neon_vld3q_dup_v:
2614 case NEON::BI__builtin_neon_vld4_dup_v:
2615 case NEON::BI__builtin_neon_vld4q_dup_v:
2616 // Get the alignment for the argument in addition to the value;
2617 // we'll use it later.
2618 PtrOp1 = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
2619 Ops.push_back(Elt: PtrOp1.emitRawPointer(CGF&: *this));
2620 continue;
2621 }
2622 }
2623
2624 Ops.push_back(Elt: EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E));
2625 }
2626
2627 switch (BuiltinID) {
2628 default: break;
2629
2630 case NEON::BI__builtin_neon_vget_lane_i8:
2631 case NEON::BI__builtin_neon_vget_lane_i16:
2632 case NEON::BI__builtin_neon_vget_lane_i32:
2633 case NEON::BI__builtin_neon_vget_lane_i64:
2634 case NEON::BI__builtin_neon_vget_lane_bf16:
2635 case NEON::BI__builtin_neon_vget_lane_f32:
2636 case NEON::BI__builtin_neon_vgetq_lane_i8:
2637 case NEON::BI__builtin_neon_vgetq_lane_i16:
2638 case NEON::BI__builtin_neon_vgetq_lane_i32:
2639 case NEON::BI__builtin_neon_vgetq_lane_i64:
2640 case NEON::BI__builtin_neon_vgetq_lane_bf16:
2641 case NEON::BI__builtin_neon_vgetq_lane_f32:
2642 case NEON::BI__builtin_neon_vduph_lane_bf16:
2643 case NEON::BI__builtin_neon_vduph_laneq_bf16:
2644 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
2645
2646 case NEON::BI__builtin_neon_vrndns_f32: {
2647 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2648 llvm::Type *Tys[] = {Arg->getType()};
2649 Function *F = CGM.getIntrinsic(IID: Intrinsic::roundeven, Tys);
2650 return Builder.CreateCall(Callee: F, Args: {Arg}, Name: "vrndn"); }
2651
2652 case NEON::BI__builtin_neon_vset_lane_i8:
2653 case NEON::BI__builtin_neon_vset_lane_i16:
2654 case NEON::BI__builtin_neon_vset_lane_i32:
2655 case NEON::BI__builtin_neon_vset_lane_i64:
2656 case NEON::BI__builtin_neon_vset_lane_bf16:
2657 case NEON::BI__builtin_neon_vset_lane_f32:
2658 case NEON::BI__builtin_neon_vsetq_lane_i8:
2659 case NEON::BI__builtin_neon_vsetq_lane_i16:
2660 case NEON::BI__builtin_neon_vsetq_lane_i32:
2661 case NEON::BI__builtin_neon_vsetq_lane_i64:
2662 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2663 case NEON::BI__builtin_neon_vsetq_lane_f32:
2664 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
2665
2666 case NEON::BI__builtin_neon_vsha1h_u32:
2667 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1h), Ops,
2668 name: "vsha1h");
2669 case NEON::BI__builtin_neon_vsha1cq_u32:
2670 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1c), Ops,
2671 name: "vsha1h");
2672 case NEON::BI__builtin_neon_vsha1pq_u32:
2673 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1p), Ops,
2674 name: "vsha1h");
2675 case NEON::BI__builtin_neon_vsha1mq_u32:
2676 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1m), Ops,
2677 name: "vsha1h");
2678
2679 case NEON::BI__builtin_neon_vcvth_bf16_f32:
2680 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vcvtbfp2bf), Ops,
2681 name: "vcvtbfp2bf");
2682 case NEON::BI__builtin_neon_vcvt_f16_f32:
2683 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vcvtfp2hf), Ops,
2684 name: "vcvtfp2hf");
2685 case NEON::BI__builtin_neon_vcvt_f32_f16:
2686 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vcvthf2fp), Ops,
2687 name: "vcvthf2fp");
2688
2689 // The ARM _MoveToCoprocessor builtins put the input register value as
2690 // the first argument, but the LLVM intrinsic expects it as the third one.
2691 case clang::ARM::BI_MoveToCoprocessor:
2692 case clang::ARM::BI_MoveToCoprocessor2: {
2693 Function *F = CGM.getIntrinsic(IID: BuiltinID == clang::ARM::BI_MoveToCoprocessor
2694 ? Intrinsic::arm_mcr
2695 : Intrinsic::arm_mcr2);
2696 return Builder.CreateCall(Callee: F, Args: {Ops[1], Ops[2], Ops[0],
2697 Ops[3], Ops[4], Ops[5]});
2698 }
2699 }
2700
2701 // Get the last argument, which specifies the vector type.
2702 assert(HasExtraArg);
2703 const Expr *Arg = E->getArg(Arg: E->getNumArgs()-1);
2704 std::optional<llvm::APSInt> Result =
2705 Arg->getIntegerConstantExpr(Ctx: getContext());
2706 if (!Result)
2707 return nullptr;
2708
2709 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
2710 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
2711 // Determine the overloaded type of this builtin.
2712 llvm::Type *Ty;
2713 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
2714 Ty = FloatTy;
2715 else
2716 Ty = DoubleTy;
2717
2718 // Determine whether this is an unsigned conversion or not.
2719 bool usgn = Result->getZExtValue() == 1;
2720 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
2721
2722 // Call the appropriate intrinsic.
2723 Function *F = CGM.getIntrinsic(IID: Int, Tys: Ty);
2724 return Builder.CreateCall(Callee: F, Args: Ops, Name: "vcvtr");
2725 }
2726
2727 // Determine the type of this overloaded NEON intrinsic.
2728 NeonTypeFlags Type = Result->getZExtValue();
2729 bool usgn = Type.isUnsigned();
2730 bool rightShift = false;
2731
2732 llvm::FixedVectorType *VTy =
2733 GetNeonType(CGF: this, TypeFlags: Type, HasFastHalfType: getTarget().hasFastHalfType(), V1Ty: false,
2734 AllowBFloatArgsAndRet: getTarget().hasBFloat16Type());
2735 llvm::Type *Ty = VTy;
2736 if (!Ty)
2737 return nullptr;
2738
2739 // Many NEON builtins have identical semantics and uses in ARM and
2740 // AArch64. Emit these in a single function.
2741 auto IntrinsicMap = ArrayRef(ARMSIMDIntrinsicMap);
2742 const ARMNeonVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap(
2743 IntrinsicMap, BuiltinID, MapProvenSorted&: NEONSIMDIntrinsicsProvenSorted);
2744 if (Builtin)
2745 return EmitCommonNeonBuiltinExpr(
2746 BuiltinID: Builtin->BuiltinID, LLVMIntrinsic: Builtin->LLVMIntrinsic, AltLLVMIntrinsic: Builtin->AltLLVMIntrinsic,
2747 NameHint: Builtin->NameHint, Modifier: Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
2748
2749 unsigned Int;
2750 switch (BuiltinID) {
2751 default: return nullptr;
2752 case NEON::BI__builtin_neon_vld1q_lane_v:
2753 // Handle 64-bit integer elements as a special case. Use shuffles of
2754 // one-element vectors to avoid poor code for i64 in the backend.
2755 if (VTy->getElementType()->isIntegerTy(BitWidth: 64)) {
2756 // Extract the other lane.
2757 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2758 int Lane = cast<ConstantInt>(Val: Ops[2])->getZExtValue();
2759 Value *SV = llvm::ConstantVector::get(V: ConstantInt::get(Ty: Int32Ty, V: 1-Lane));
2760 Ops[1] = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[1], Mask: SV);
2761 // Load the value as a one-element vector.
2762 Ty = llvm::FixedVectorType::get(ElementType: VTy->getElementType(), NumElts: 1);
2763 llvm::Type *Tys[] = {Ty, Int8PtrTy};
2764 Function *F = CGM.getIntrinsic(IID: Intrinsic::arm_neon_vld1, Tys);
2765 Value *Align = getAlignmentValue32(PtrOp0);
2766 Value *Ld = Builder.CreateCall(Callee: F, Args: {Ops[0], Align});
2767 // Combine them.
2768 int Indices[] = {1 - Lane, Lane};
2769 return Builder.CreateShuffleVector(V1: Ops[1], V2: Ld, Mask: Indices, Name: "vld1q_lane");
2770 }
2771 [[fallthrough]];
2772 case NEON::BI__builtin_neon_vld1_lane_v: {
2773 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2774 PtrOp0 = PtrOp0.withElementType(ElemTy: VTy->getElementType());
2775 Value *Ld = Builder.CreateLoad(Addr: PtrOp0);
2776 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ld, Idx: Ops[2], Name: "vld1_lane");
2777 }
2778 case NEON::BI__builtin_neon_vqrshrn_n_v:
2779 Int =
2780 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
2781 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqrshrn_n",
2782 shift: 1, rightshift: true);
2783 case NEON::BI__builtin_neon_vqrshrun_n_v:
2784 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vqrshiftnsu, Tys: Ty),
2785 Ops, name: "vqrshrun_n", shift: 1, rightshift: true);
2786 case NEON::BI__builtin_neon_vqshrn_n_v:
2787 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
2788 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshrn_n",
2789 shift: 1, rightshift: true);
2790 case NEON::BI__builtin_neon_vqshrun_n_v:
2791 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vqshiftnsu, Tys: Ty),
2792 Ops, name: "vqshrun_n", shift: 1, rightshift: true);
2793 case NEON::BI__builtin_neon_vrecpe_v:
2794 case NEON::BI__builtin_neon_vrecpeq_v:
2795 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vrecpe, Tys: Ty),
2796 Ops, name: "vrecpe");
2797 case NEON::BI__builtin_neon_vrshrn_n_v:
2798 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vrshiftn, Tys: Ty),
2799 Ops, name: "vrshrn_n", shift: 1, rightshift: true);
2800 case NEON::BI__builtin_neon_vrsra_n_v:
2801 case NEON::BI__builtin_neon_vrsraq_n_v:
2802 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
2803 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2804 Ops[2] = EmitNeonShiftVector(V: Ops[2], Ty, neg: true);
2805 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
2806 Ops[1] = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Int, Tys: Ty), Args: {Ops[1], Ops[2]});
2807 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1], Name: "vrsra_n");
2808 case NEON::BI__builtin_neon_vsri_n_v:
2809 case NEON::BI__builtin_neon_vsriq_n_v:
2810 rightShift = true;
2811 [[fallthrough]];
2812 case NEON::BI__builtin_neon_vsli_n_v:
2813 case NEON::BI__builtin_neon_vsliq_n_v:
2814 Ops[2] = EmitNeonShiftVector(V: Ops[2], Ty, neg: rightShift);
2815 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vshiftins, Tys: Ty),
2816 Ops, name: "vsli_n");
2817 case NEON::BI__builtin_neon_vsra_n_v:
2818 case NEON::BI__builtin_neon_vsraq_n_v:
2819 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
2820 Ops[1] = EmitNeonRShiftImm(Vec: Ops[1], Shift: Ops[2], Ty, usgn, name: "vsra_n");
2821 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
2822 case NEON::BI__builtin_neon_vst1q_lane_v:
2823 // Handle 64-bit integer elements as a special case. Use a shuffle to get
2824 // a one-element vector and avoid poor code for i64 in the backend.
2825 if (VTy->getElementType()->isIntegerTy(BitWidth: 64)) {
2826 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2827 Value *SV = llvm::ConstantVector::get(V: cast<llvm::Constant>(Val: Ops[2]));
2828 Ops[1] = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[1], Mask: SV);
2829 Ops[2] = getAlignmentValue32(PtrOp0);
2830 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
2831 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vst1,
2832 Tys), Args: Ops);
2833 }
2834 [[fallthrough]];
2835 case NEON::BI__builtin_neon_vst1_lane_v: {
2836 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2837 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2]);
2838 return Builder.CreateStore(Val: Ops[1],
2839 Addr: PtrOp0.withElementType(ElemTy: Ops[1]->getType()));
2840 }
2841 case NEON::BI__builtin_neon_vtbl1_v:
2842 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl1),
2843 Ops, name: "vtbl1");
2844 case NEON::BI__builtin_neon_vtbl2_v:
2845 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl2),
2846 Ops, name: "vtbl2");
2847 case NEON::BI__builtin_neon_vtbl3_v:
2848 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl3),
2849 Ops, name: "vtbl3");
2850 case NEON::BI__builtin_neon_vtbl4_v:
2851 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl4),
2852 Ops, name: "vtbl4");
2853 case NEON::BI__builtin_neon_vtbx1_v:
2854 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx1),
2855 Ops, name: "vtbx1");
2856 case NEON::BI__builtin_neon_vtbx2_v:
2857 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx2),
2858 Ops, name: "vtbx2");
2859 case NEON::BI__builtin_neon_vtbx3_v:
2860 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx3),
2861 Ops, name: "vtbx3");
2862 case NEON::BI__builtin_neon_vtbx4_v:
2863 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx4),
2864 Ops, name: "vtbx4");
2865 }
2866}
2867
2868template<typename Integer>
2869static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context) {
2870 return E->getIntegerConstantExpr(Ctx: Context)->getExtValue();
2871}
2872
2873static llvm::Value *SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V,
2874 llvm::Type *T, bool Unsigned) {
2875 // Helper function called by Tablegen-constructed ARM MVE builtin codegen,
2876 // which finds it convenient to specify signed/unsigned as a boolean flag.
2877 return Unsigned ? Builder.CreateZExt(V, DestTy: T) : Builder.CreateSExt(V, DestTy: T);
2878}
2879
2880static llvm::Value *MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V,
2881 uint32_t Shift, bool Unsigned) {
2882 // MVE helper function for integer shift right. This must handle signed vs
2883 // unsigned, and also deal specially with the case where the shift count is
2884 // equal to the lane size. In LLVM IR, an LShr with that parameter would be
2885 // undefined behavior, but in MVE it's legal, so we must convert it to code
2886 // that is not undefined in IR.
2887 unsigned LaneBits = cast<llvm::VectorType>(Val: V->getType())
2888 ->getElementType()
2889 ->getPrimitiveSizeInBits();
2890 if (Shift == LaneBits) {
2891 // An unsigned shift of the full lane size always generates zero, so we can
2892 // simply emit a zero vector. A signed shift of the full lane size does the
2893 // same thing as shifting by one bit fewer.
2894 if (Unsigned)
2895 return llvm::Constant::getNullValue(Ty: V->getType());
2896 else
2897 --Shift;
2898 }
2899 return Unsigned ? Builder.CreateLShr(LHS: V, RHS: Shift) : Builder.CreateAShr(LHS: V, RHS: Shift);
2900}
2901
2902static llvm::Value *ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V) {
2903 // MVE-specific helper function for a vector splat, which infers the element
2904 // count of the output vector by knowing that MVE vectors are all 128 bits
2905 // wide.
2906 unsigned Elements = 128 / V->getType()->getPrimitiveSizeInBits();
2907 return Builder.CreateVectorSplat(NumElts: Elements, V);
2908}
2909
2910static llvm::Value *ARMMVEVectorReinterpret(CGBuilderTy &Builder,
2911 CodeGenFunction *CGF,
2912 llvm::Value *V,
2913 llvm::Type *DestType) {
2914 // Convert one MVE vector type into another by reinterpreting its in-register
2915 // format.
2916 //
2917 // Little-endian, this is identical to a bitcast (which reinterprets the
2918 // memory format). But big-endian, they're not necessarily the same, because
2919 // the register and memory formats map to each other differently depending on
2920 // the lane size.
2921 //
2922 // We generate a bitcast whenever we can (if we're little-endian, or if the
2923 // lane sizes are the same anyway). Otherwise we fall back to an IR intrinsic
2924 // that performs the different kind of reinterpretation.
2925 if (CGF->getTarget().isBigEndian() &&
2926 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
2927 return Builder.CreateCall(
2928 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vreinterpretq,
2929 Tys: {DestType, V->getType()}),
2930 Args: V);
2931 } else {
2932 return Builder.CreateBitCast(V, DestTy: DestType);
2933 }
2934}
2935
2936static llvm::Value *VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd) {
2937 // Make a shufflevector that extracts every other element of a vector (evens
2938 // or odds, as desired).
2939 SmallVector<int, 16> Indices;
2940 unsigned InputElements =
2941 cast<llvm::FixedVectorType>(Val: V->getType())->getNumElements();
2942 for (unsigned i = 0; i < InputElements; i += 2)
2943 Indices.push_back(Elt: i + Odd);
2944 return Builder.CreateShuffleVector(V, Mask: Indices);
2945}
2946
2947static llvm::Value *VectorZip(CGBuilderTy &Builder, llvm::Value *V0,
2948 llvm::Value *V1) {
2949 // Make a shufflevector that interleaves two vectors element by element.
2950 assert(V0->getType() == V1->getType() && "Can't zip different vector types");
2951 SmallVector<int, 16> Indices;
2952 unsigned InputElements =
2953 cast<llvm::FixedVectorType>(Val: V0->getType())->getNumElements();
2954 for (unsigned i = 0; i < InputElements; i++) {
2955 Indices.push_back(Elt: i);
2956 Indices.push_back(Elt: i + InputElements);
2957 }
2958 return Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: Indices);
2959}
2960
2961template<unsigned HighBit, unsigned OtherBits>
2962static llvm::Value *ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT) {
2963 // MVE-specific helper function to make a vector splat of a constant such as
2964 // UINT_MAX or INT_MIN, in which all bits below the highest one are equal.
2965 llvm::Type *T = cast<llvm::VectorType>(Val: VT)->getElementType();
2966 unsigned LaneBits = T->getPrimitiveSizeInBits();
2967 uint32_t Value = HighBit << (LaneBits - 1);
2968 if (OtherBits)
2969 Value |= (1UL << (LaneBits - 1)) - 1;
2970 llvm::Value *Lane = llvm::ConstantInt::get(Ty: T, V: Value);
2971 return ARMMVEVectorSplat(Builder, V: Lane);
2972}
2973
2974static llvm::Value *ARMMVEVectorElementReverse(CGBuilderTy &Builder,
2975 llvm::Value *V,
2976 unsigned ReverseWidth) {
2977 // MVE-specific helper function which reverses the elements of a
2978 // vector within every (ReverseWidth)-bit collection of lanes.
2979 SmallVector<int, 16> Indices;
2980 unsigned LaneSize = V->getType()->getScalarSizeInBits();
2981 unsigned Elements = 128 / LaneSize;
2982 unsigned Mask = ReverseWidth / LaneSize - 1;
2983 for (unsigned i = 0; i < Elements; i++)
2984 Indices.push_back(Elt: i ^ Mask);
2985 return Builder.CreateShuffleVector(V, Mask: Indices);
2986}
2987
2988static llvm::Value *ARMMVECreateSIToFP(CGBuilderTy &Builder,
2989 CodeGenFunction *CGF, llvm::Value *V,
2990 llvm::Type *Ty) {
2991 return Builder.CreateCall(
2992 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_fp_int, Tys: {Ty, V->getType()}),
2993 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)});
2994}
2995
2996static llvm::Value *ARMMVECreateUIToFP(CGBuilderTy &Builder,
2997 CodeGenFunction *CGF, llvm::Value *V,
2998 llvm::Type *Ty) {
2999 return Builder.CreateCall(
3000 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_fp_int, Tys: {Ty, V->getType()}),
3001 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1)});
3002}
3003
3004static llvm::Value *ARMMVECreateFPToSI(CGBuilderTy &Builder,
3005 CodeGenFunction *CGF, llvm::Value *V,
3006 llvm::Type *Ty) {
3007 return Builder.CreateCall(
3008 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_int_fp, Tys: {Ty, V->getType()}),
3009 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)});
3010}
3011
3012static llvm::Value *ARMMVECreateFPToUI(CGBuilderTy &Builder,
3013 CodeGenFunction *CGF, llvm::Value *V,
3014 llvm::Type *Ty) {
3015 return Builder.CreateCall(
3016 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_int_fp, Tys: {Ty, V->getType()}),
3017 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1)});
3018}
3019
3020Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID,
3021 const CallExpr *E,
3022 ReturnValueSlot ReturnValue,
3023 llvm::Triple::ArchType Arch) {
3024 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
3025 Intrinsic::ID IRIntr;
3026 unsigned NumVectors;
3027
3028 // Code autogenerated by Tablegen will handle all the simple builtins.
3029 switch (BuiltinID) {
3030 #include "clang/Basic/arm_mve_builtin_cg.inc"
3031
3032 // If we didn't match an MVE builtin id at all, go back to the
3033 // main EmitARMBuiltinExpr.
3034 default:
3035 return nullptr;
3036 }
3037
3038 // Anything that breaks from that switch is an MVE builtin that
3039 // needs handwritten code to generate.
3040
3041 switch (CustomCodeGenType) {
3042
3043 case CustomCodeGen::VLD24: {
3044 llvm::SmallVector<Value *, 4> Ops;
3045 llvm::SmallVector<llvm::Type *, 4> Tys;
3046
3047 auto MvecCType = E->getType();
3048 auto MvecLType = ConvertType(T: MvecCType);
3049 assert(MvecLType->isStructTy() &&
3050 "Return type for vld[24]q should be a struct");
3051 assert(MvecLType->getStructNumElements() == 1 &&
3052 "Return-type struct for vld[24]q should have one element");
3053 auto MvecLTypeInner = MvecLType->getStructElementType(N: 0);
3054 assert(MvecLTypeInner->isArrayTy() &&
3055 "Return-type struct for vld[24]q should contain an array");
3056 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3057 "Array member of return-type struct vld[24]q has wrong length");
3058 auto VecLType = MvecLTypeInner->getArrayElementType();
3059
3060 Tys.push_back(Elt: VecLType);
3061
3062 auto Addr = E->getArg(Arg: 0);
3063 Ops.push_back(Elt: EmitScalarExpr(E: Addr));
3064 Tys.push_back(Elt: ConvertType(T: Addr->getType()));
3065
3066 Function *F = CGM.getIntrinsic(IID: IRIntr, Tys: ArrayRef(Tys));
3067 Value *LoadResult = Builder.CreateCall(Callee: F, Args: Ops);
3068 Value *MvecOut = PoisonValue::get(T: MvecLType);
3069 for (unsigned i = 0; i < NumVectors; ++i) {
3070 Value *Vec = Builder.CreateExtractValue(Agg: LoadResult, Idxs: i);
3071 MvecOut = Builder.CreateInsertValue(Agg: MvecOut, Val: Vec, Idxs: {0, i});
3072 }
3073
3074 if (ReturnValue.isNull())
3075 return MvecOut;
3076 else
3077 return Builder.CreateStore(Val: MvecOut, Addr: ReturnValue.getAddress());
3078 }
3079
3080 case CustomCodeGen::VST24: {
3081 llvm::SmallVector<Value *, 4> Ops;
3082 llvm::SmallVector<llvm::Type *, 4> Tys;
3083
3084 auto Addr = E->getArg(Arg: 0);
3085 Ops.push_back(Elt: EmitScalarExpr(E: Addr));
3086 Tys.push_back(Elt: ConvertType(T: Addr->getType()));
3087
3088 auto MvecCType = E->getArg(Arg: 1)->getType();
3089 auto MvecLType = ConvertType(T: MvecCType);
3090 assert(MvecLType->isStructTy() && "Data type for vst2q should be a struct");
3091 assert(MvecLType->getStructNumElements() == 1 &&
3092 "Data-type struct for vst2q should have one element");
3093 auto MvecLTypeInner = MvecLType->getStructElementType(N: 0);
3094 assert(MvecLTypeInner->isArrayTy() &&
3095 "Data-type struct for vst2q should contain an array");
3096 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3097 "Array member of return-type struct vld[24]q has wrong length");
3098 auto VecLType = MvecLTypeInner->getArrayElementType();
3099
3100 Tys.push_back(Elt: VecLType);
3101
3102 AggValueSlot MvecSlot = CreateAggTemp(T: MvecCType);
3103 EmitAggExpr(E: E->getArg(Arg: 1), AS: MvecSlot);
3104 auto Mvec = Builder.CreateLoad(Addr: MvecSlot.getAddress());
3105 for (unsigned i = 0; i < NumVectors; i++)
3106 Ops.push_back(Elt: Builder.CreateExtractValue(Agg: Mvec, Idxs: {0, i}));
3107
3108 Function *F = CGM.getIntrinsic(IID: IRIntr, Tys: ArrayRef(Tys));
3109 Value *ToReturn = nullptr;
3110 for (unsigned i = 0; i < NumVectors; i++) {
3111 Ops.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: i));
3112 ToReturn = Builder.CreateCall(Callee: F, Args: Ops);
3113 Ops.pop_back();
3114 }
3115 return ToReturn;
3116 }
3117 }
3118 llvm_unreachable("unknown custom codegen type.");
3119}
3120
3121Value *CodeGenFunction::EmitARMCDEBuiltinExpr(unsigned BuiltinID,
3122 const CallExpr *E,
3123 ReturnValueSlot ReturnValue,
3124 llvm::Triple::ArchType Arch) {
3125 switch (BuiltinID) {
3126 default:
3127 return nullptr;
3128#include "clang/Basic/arm_cde_builtin_cg.inc"
3129 }
3130}
3131
3132static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
3133 const CallExpr *E,
3134 SmallVectorImpl<Value *> &Ops,
3135 llvm::Triple::ArchType Arch) {
3136 unsigned int Int = 0;
3137 const char *s = nullptr;
3138
3139 switch (BuiltinID) {
3140 default:
3141 return nullptr;
3142 case NEON::BI__builtin_neon_vtbl1_v:
3143 case NEON::BI__builtin_neon_vqtbl1_v:
3144 case NEON::BI__builtin_neon_vqtbl1q_v:
3145 case NEON::BI__builtin_neon_vtbl2_v:
3146 case NEON::BI__builtin_neon_vqtbl2_v:
3147 case NEON::BI__builtin_neon_vqtbl2q_v:
3148 case NEON::BI__builtin_neon_vtbl3_v:
3149 case NEON::BI__builtin_neon_vqtbl3_v:
3150 case NEON::BI__builtin_neon_vqtbl3q_v:
3151 case NEON::BI__builtin_neon_vtbl4_v:
3152 case NEON::BI__builtin_neon_vqtbl4_v:
3153 case NEON::BI__builtin_neon_vqtbl4q_v:
3154 break;
3155 case NEON::BI__builtin_neon_vtbx1_v:
3156 case NEON::BI__builtin_neon_vqtbx1_v:
3157 case NEON::BI__builtin_neon_vqtbx1q_v:
3158 case NEON::BI__builtin_neon_vtbx2_v:
3159 case NEON::BI__builtin_neon_vqtbx2_v:
3160 case NEON::BI__builtin_neon_vqtbx2q_v:
3161 case NEON::BI__builtin_neon_vtbx3_v:
3162 case NEON::BI__builtin_neon_vqtbx3_v:
3163 case NEON::BI__builtin_neon_vqtbx3q_v:
3164 case NEON::BI__builtin_neon_vtbx4_v:
3165 case NEON::BI__builtin_neon_vqtbx4_v:
3166 case NEON::BI__builtin_neon_vqtbx4q_v:
3167 break;
3168 }
3169
3170 assert(E->getNumArgs() >= 3);
3171
3172 // Get the last argument, which specifies the vector type.
3173 const Expr *Arg = E->getArg(Arg: E->getNumArgs() - 1);
3174 std::optional<llvm::APSInt> Result =
3175 Arg->getIntegerConstantExpr(Ctx: CGF.getContext());
3176 if (!Result)
3177 return nullptr;
3178
3179 // Determine the type of this overloaded NEON intrinsic.
3180 NeonTypeFlags Type = Result->getZExtValue();
3181 llvm::FixedVectorType *Ty = GetNeonType(CGF: &CGF, TypeFlags: Type);
3182 if (!Ty)
3183 return nullptr;
3184
3185 CodeGen::CGBuilderTy &Builder = CGF.Builder;
3186
3187 // AArch64 scalar builtins are not overloaded, they do not have an extra
3188 // argument that specifies the vector type, need to handle each case.
3189 switch (BuiltinID) {
3190 case NEON::BI__builtin_neon_vtbl1_v: {
3191 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 1), ExtOp: nullptr, IndexOp: Ops[1],
3192 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl1, Name: "vtbl1");
3193 }
3194 case NEON::BI__builtin_neon_vtbl2_v: {
3195 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 2), ExtOp: nullptr, IndexOp: Ops[2],
3196 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl1, Name: "vtbl1");
3197 }
3198 case NEON::BI__builtin_neon_vtbl3_v: {
3199 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 3), ExtOp: nullptr, IndexOp: Ops[3],
3200 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl2, Name: "vtbl2");
3201 }
3202 case NEON::BI__builtin_neon_vtbl4_v: {
3203 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 4), ExtOp: nullptr, IndexOp: Ops[4],
3204 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl2, Name: "vtbl2");
3205 }
3206 case NEON::BI__builtin_neon_vtbx1_v: {
3207 Value *TblRes =
3208 packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 1), ExtOp: nullptr, IndexOp: Ops[2], ResTy: Ty,
3209 IntID: Intrinsic::aarch64_neon_tbl1, Name: "vtbl1");
3210
3211 llvm::Constant *EightV = ConstantInt::get(Ty, V: 8);
3212 Value *CmpRes = Builder.CreateICmp(P: ICmpInst::ICMP_UGE, LHS: Ops[2], RHS: EightV);
3213 CmpRes = Builder.CreateSExt(V: CmpRes, DestTy: Ty);
3214
3215 Value *EltsFromInput = Builder.CreateAnd(LHS: CmpRes, RHS: Ops[0]);
3216 Value *EltsFromTbl = Builder.CreateAnd(LHS: Builder.CreateNot(V: CmpRes), RHS: TblRes);
3217 return Builder.CreateOr(LHS: EltsFromInput, RHS: EltsFromTbl, Name: "vtbx");
3218 }
3219 case NEON::BI__builtin_neon_vtbx2_v: {
3220 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 2), ExtOp: Ops[0], IndexOp: Ops[3],
3221 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbx1, Name: "vtbx1");
3222 }
3223 case NEON::BI__builtin_neon_vtbx3_v: {
3224 Value *TblRes =
3225 packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 3), ExtOp: nullptr, IndexOp: Ops[4], ResTy: Ty,
3226 IntID: Intrinsic::aarch64_neon_tbl2, Name: "vtbl2");
3227
3228 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, V: 24);
3229 Value *CmpRes = Builder.CreateICmp(P: ICmpInst::ICMP_UGE, LHS: Ops[4],
3230 RHS: TwentyFourV);
3231 CmpRes = Builder.CreateSExt(V: CmpRes, DestTy: Ty);
3232
3233 Value *EltsFromInput = Builder.CreateAnd(LHS: CmpRes, RHS: Ops[0]);
3234 Value *EltsFromTbl = Builder.CreateAnd(LHS: Builder.CreateNot(V: CmpRes), RHS: TblRes);
3235 return Builder.CreateOr(LHS: EltsFromInput, RHS: EltsFromTbl, Name: "vtbx");
3236 }
3237 case NEON::BI__builtin_neon_vtbx4_v: {
3238 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 4), ExtOp: Ops[0], IndexOp: Ops[5],
3239 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbx2, Name: "vtbx2");
3240 }
3241 case NEON::BI__builtin_neon_vqtbl1_v:
3242 case NEON::BI__builtin_neon_vqtbl1q_v:
3243 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
3244 case NEON::BI__builtin_neon_vqtbl2_v:
3245 case NEON::BI__builtin_neon_vqtbl2q_v: {
3246 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
3247 case NEON::BI__builtin_neon_vqtbl3_v:
3248 case NEON::BI__builtin_neon_vqtbl3q_v:
3249 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
3250 case NEON::BI__builtin_neon_vqtbl4_v:
3251 case NEON::BI__builtin_neon_vqtbl4q_v:
3252 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
3253 case NEON::BI__builtin_neon_vqtbx1_v:
3254 case NEON::BI__builtin_neon_vqtbx1q_v:
3255 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
3256 case NEON::BI__builtin_neon_vqtbx2_v:
3257 case NEON::BI__builtin_neon_vqtbx2q_v:
3258 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
3259 case NEON::BI__builtin_neon_vqtbx3_v:
3260 case NEON::BI__builtin_neon_vqtbx3q_v:
3261 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
3262 case NEON::BI__builtin_neon_vqtbx4_v:
3263 case NEON::BI__builtin_neon_vqtbx4q_v:
3264 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
3265 }
3266 }
3267
3268 if (!Int)
3269 return nullptr;
3270
3271 Function *F = CGF.CGM.getIntrinsic(IID: Int, Tys: Ty);
3272 return CGF.EmitNeonCall(F, Ops, name: s);
3273}
3274
3275Value *CodeGenFunction::vectorWrapScalar16(Value *Op) {
3276 auto *VTy = llvm::FixedVectorType::get(ElementType: Int16Ty, NumElts: 4);
3277 Op = Builder.CreateBitCast(V: Op, DestTy: Int16Ty);
3278 Value *V = PoisonValue::get(T: VTy);
3279 llvm::Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
3280 Op = Builder.CreateInsertElement(Vec: V, NewElt: Op, Idx: CI);
3281 return Op;
3282}
3283
3284/// SVEBuiltinMemEltTy - Returns the memory element type for this memory
3285/// access builtin. Only required if it can't be inferred from the base pointer
3286/// operand.
3287llvm::Type *CodeGenFunction::SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags) {
3288 switch (TypeFlags.getMemEltType()) {
3289 case SVETypeFlags::MemEltTyDefault:
3290 return getEltType(TypeFlags);
3291 case SVETypeFlags::MemEltTyInt8:
3292 return Builder.getInt8Ty();
3293 case SVETypeFlags::MemEltTyInt16:
3294 return Builder.getInt16Ty();
3295 case SVETypeFlags::MemEltTyInt32:
3296 return Builder.getInt32Ty();
3297 case SVETypeFlags::MemEltTyInt64:
3298 return Builder.getInt64Ty();
3299 }
3300 llvm_unreachable("Unknown MemEltType");
3301}
3302
3303llvm::Type *CodeGenFunction::getEltType(const SVETypeFlags &TypeFlags) {
3304 switch (TypeFlags.getEltType()) {
3305 default:
3306 llvm_unreachable("Invalid SVETypeFlag!");
3307
3308 case SVETypeFlags::EltTyMFloat8:
3309 case SVETypeFlags::EltTyInt8:
3310 return Builder.getInt8Ty();
3311 case SVETypeFlags::EltTyInt16:
3312 return Builder.getInt16Ty();
3313 case SVETypeFlags::EltTyInt32:
3314 return Builder.getInt32Ty();
3315 case SVETypeFlags::EltTyInt64:
3316 return Builder.getInt64Ty();
3317 case SVETypeFlags::EltTyInt128:
3318 return Builder.getInt128Ty();
3319
3320 case SVETypeFlags::EltTyFloat16:
3321 return Builder.getHalfTy();
3322 case SVETypeFlags::EltTyFloat32:
3323 return Builder.getFloatTy();
3324 case SVETypeFlags::EltTyFloat64:
3325 return Builder.getDoubleTy();
3326
3327 case SVETypeFlags::EltTyBFloat16:
3328 return Builder.getBFloatTy();
3329
3330 case SVETypeFlags::EltTyBool8:
3331 case SVETypeFlags::EltTyBool16:
3332 case SVETypeFlags::EltTyBool32:
3333 case SVETypeFlags::EltTyBool64:
3334 return Builder.getInt1Ty();
3335 }
3336}
3337
3338// Return the llvm predicate vector type corresponding to the specified element
3339// TypeFlags.
3340llvm::ScalableVectorType *
3341CodeGenFunction::getSVEPredType(const SVETypeFlags &TypeFlags) {
3342 switch (TypeFlags.getEltType()) {
3343 default: llvm_unreachable("Unhandled SVETypeFlag!");
3344
3345 case SVETypeFlags::EltTyInt8:
3346 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 16);
3347 case SVETypeFlags::EltTyInt16:
3348 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3349 case SVETypeFlags::EltTyInt32:
3350 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3351 case SVETypeFlags::EltTyInt64:
3352 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3353
3354 case SVETypeFlags::EltTyBFloat16:
3355 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3356 case SVETypeFlags::EltTyFloat16:
3357 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3358 case SVETypeFlags::EltTyFloat32:
3359 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3360 case SVETypeFlags::EltTyFloat64:
3361 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3362
3363 case SVETypeFlags::EltTyBool8:
3364 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 16);
3365 case SVETypeFlags::EltTyBool16:
3366 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3367 case SVETypeFlags::EltTyBool32:
3368 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3369 case SVETypeFlags::EltTyBool64:
3370 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3371 }
3372}
3373
3374// Return the llvm vector type corresponding to the specified element TypeFlags.
3375llvm::ScalableVectorType *
3376CodeGenFunction::getSVEType(const SVETypeFlags &TypeFlags) {
3377 switch (TypeFlags.getEltType()) {
3378 default:
3379 llvm_unreachable("Invalid SVETypeFlag!");
3380
3381 case SVETypeFlags::EltTyInt8:
3382 return llvm::ScalableVectorType::get(ElementType: Builder.getInt8Ty(), MinNumElts: 16);
3383 case SVETypeFlags::EltTyInt16:
3384 return llvm::ScalableVectorType::get(ElementType: Builder.getInt16Ty(), MinNumElts: 8);
3385 case SVETypeFlags::EltTyInt32:
3386 return llvm::ScalableVectorType::get(ElementType: Builder.getInt32Ty(), MinNumElts: 4);
3387 case SVETypeFlags::EltTyInt64:
3388 return llvm::ScalableVectorType::get(ElementType: Builder.getInt64Ty(), MinNumElts: 2);
3389
3390 case SVETypeFlags::EltTyMFloat8:
3391 return llvm::ScalableVectorType::get(ElementType: Builder.getInt8Ty(), MinNumElts: 16);
3392 case SVETypeFlags::EltTyFloat16:
3393 return llvm::ScalableVectorType::get(ElementType: Builder.getHalfTy(), MinNumElts: 8);
3394 case SVETypeFlags::EltTyBFloat16:
3395 return llvm::ScalableVectorType::get(ElementType: Builder.getBFloatTy(), MinNumElts: 8);
3396 case SVETypeFlags::EltTyFloat32:
3397 return llvm::ScalableVectorType::get(ElementType: Builder.getFloatTy(), MinNumElts: 4);
3398 case SVETypeFlags::EltTyFloat64:
3399 return llvm::ScalableVectorType::get(ElementType: Builder.getDoubleTy(), MinNumElts: 2);
3400
3401 case SVETypeFlags::EltTyBool8:
3402 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 16);
3403 case SVETypeFlags::EltTyBool16:
3404 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3405 case SVETypeFlags::EltTyBool32:
3406 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3407 case SVETypeFlags::EltTyBool64:
3408 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3409 }
3410}
3411
3412constexpr unsigned SVEBitsPerBlock = 128;
3413
3414static llvm::ScalableVectorType *getSVEVectorForElementType(llvm::Type *EltTy) {
3415 unsigned NumElts = SVEBitsPerBlock / EltTy->getScalarSizeInBits();
3416 return llvm::ScalableVectorType::get(ElementType: EltTy, MinNumElts: NumElts);
3417}
3418
3419// Reinterpret the input predicate so that it can be used to correctly isolate
3420// the elements of the specified datatype.
3421Value *CodeGenFunction::EmitSVEPredicateCast(Value *Pred,
3422 llvm::ScalableVectorType *VTy) {
3423
3424 if (isa<TargetExtType>(Val: Pred->getType()) &&
3425 cast<TargetExtType>(Val: Pred->getType())->getName() == "aarch64.svcount")
3426 return Pred;
3427
3428 auto *RTy = llvm::VectorType::get(ElementType: IntegerType::get(C&: getLLVMContext(), NumBits: 1), Other: VTy);
3429 if (Pred->getType() == RTy)
3430 return Pred;
3431
3432 unsigned IntID;
3433 llvm::Type *IntrinsicTy;
3434 switch (VTy->getMinNumElements()) {
3435 default:
3436 llvm_unreachable("unsupported element count!");
3437 case 1:
3438 case 2:
3439 case 4:
3440 case 8:
3441 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
3442 IntrinsicTy = RTy;
3443 break;
3444 case 16:
3445 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
3446 IntrinsicTy = Pred->getType();
3447 break;
3448 }
3449
3450 Function *F = CGM.getIntrinsic(IID: IntID, Tys: IntrinsicTy);
3451 Value *C = Builder.CreateCall(Callee: F, Args: Pred);
3452 assert(C->getType() == RTy && "Unexpected return type!");
3453 return C;
3454}
3455
3456Value *CodeGenFunction::EmitSVEPredicateTupleCast(Value *PredTuple,
3457 llvm::StructType *Ty) {
3458 if (PredTuple->getType() == Ty)
3459 return PredTuple;
3460
3461 Value *Ret = llvm::PoisonValue::get(T: Ty);
3462 for (unsigned I = 0; I < Ty->getNumElements(); ++I) {
3463 Value *Pred = Builder.CreateExtractValue(Agg: PredTuple, Idxs: I);
3464 Pred = EmitSVEPredicateCast(
3465 Pred, VTy: cast<llvm::ScalableVectorType>(Val: Ty->getTypeAtIndex(N: I)));
3466 Ret = Builder.CreateInsertValue(Agg: Ret, Val: Pred, Idxs: I);
3467 }
3468
3469 return Ret;
3470}
3471
3472Value *CodeGenFunction::EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
3473 SmallVectorImpl<Value *> &Ops,
3474 unsigned IntID) {
3475 auto *ResultTy = getSVEType(TypeFlags);
3476 auto *OverloadedTy =
3477 llvm::ScalableVectorType::get(ElementType: SVEBuiltinMemEltTy(TypeFlags), SVTy: ResultTy);
3478 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {OverloadedTy, Ops[1]->getType()});
3479
3480 // At the ACLE level there's only one predicate type, svbool_t, which is
3481 // mapped to <n x 16 x i1>. However, this might be incompatible with the
3482 // actual type being loaded. For example, when loading doubles (i64) the
3483 // predicate should be <n x 2 x i1> instead. At the IR level the type of
3484 // the predicate and the data being loaded must match. Cast to the type
3485 // expected by the intrinsic. The intrinsic itself should be defined in
3486 // a way than enforces relations between parameter types.
3487 Ops[0] = EmitSVEPredicateCast(
3488 Pred: Ops[0], VTy: cast<llvm::ScalableVectorType>(Val: F->getArg(i: 0)->getType()));
3489
3490 // Pass 0 when the offset is missing. This can only be applied when using
3491 // the "vector base" addressing mode for which ACLE allows no offset. The
3492 // corresponding LLVM IR always requires an offset.
3493 if (Ops.size() == 2) {
3494 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
3495 Ops.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: 0));
3496 }
3497
3498 // For "vector base, scalar index" scale the index so that it becomes a
3499 // scalar offset.
3500 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
3501 unsigned BytesPerElt =
3502 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
3503 Ops[2] = Builder.CreateShl(LHS: Ops[2], RHS: Log2_32(Value: BytesPerElt));
3504 }
3505
3506 Value *Call = Builder.CreateCall(Callee: F, Args: Ops);
3507
3508 // The following sext/zext is only needed when ResultTy != OverloadedTy. In
3509 // other cases it's folded into a nop.
3510 return TypeFlags.isZExtReturn() ? Builder.CreateZExt(V: Call, DestTy: ResultTy)
3511 : Builder.CreateSExt(V: Call, DestTy: ResultTy);
3512}
3513
3514Value *CodeGenFunction::EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
3515 SmallVectorImpl<Value *> &Ops,
3516 unsigned IntID) {
3517 auto *SrcDataTy = getSVEType(TypeFlags);
3518 auto *OverloadedTy =
3519 llvm::ScalableVectorType::get(ElementType: SVEBuiltinMemEltTy(TypeFlags), SVTy: SrcDataTy);
3520
3521 // In ACLE the source data is passed in the last argument, whereas in LLVM IR
3522 // it's the first argument. Move it accordingly.
3523 Ops.insert(I: Ops.begin(), Elt: Ops.pop_back_val());
3524
3525 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {OverloadedTy, Ops[2]->getType()});
3526
3527 // Pass 0 when the offset is missing. This can only be applied when using
3528 // the "vector base" addressing mode for which ACLE allows no offset. The
3529 // corresponding LLVM IR always requires an offset.
3530 if (Ops.size() == 3) {
3531 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
3532 Ops.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: 0));
3533 }
3534
3535 // Truncation is needed when SrcDataTy != OverloadedTy. In other cases it's
3536 // folded into a nop.
3537 Ops[0] = Builder.CreateTrunc(V: Ops[0], DestTy: OverloadedTy);
3538
3539 // At the ACLE level there's only one predicate type, svbool_t, which is
3540 // mapped to <n x 16 x i1>. However, this might be incompatible with the
3541 // actual type being stored. For example, when storing doubles (i64) the
3542 // predicated should be <n x 2 x i1> instead. At the IR level the type of
3543 // the predicate and the data being stored must match. Cast to the type
3544 // expected by the intrinsic. The intrinsic itself should be defined in
3545 // a way that enforces relations between parameter types.
3546 Ops[1] = EmitSVEPredicateCast(
3547 Pred: Ops[1], VTy: cast<llvm::ScalableVectorType>(Val: F->getArg(i: 1)->getType()));
3548
3549 // For "vector base, scalar index" scale the index so that it becomes a
3550 // scalar offset.
3551 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
3552 unsigned BytesPerElt =
3553 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
3554 Ops[3] = Builder.CreateShl(LHS: Ops[3], RHS: Log2_32(Value: BytesPerElt));
3555 }
3556
3557 return Builder.CreateCall(Callee: F, Args: Ops);
3558}
3559
3560Value *CodeGenFunction::EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
3561 SmallVectorImpl<Value *> &Ops,
3562 unsigned IntID) {
3563 // The gather prefetches are overloaded on the vector input - this can either
3564 // be the vector of base addresses or vector of offsets.
3565 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Val: Ops[1]->getType());
3566 if (!OverloadedTy)
3567 OverloadedTy = cast<llvm::ScalableVectorType>(Val: Ops[2]->getType());
3568
3569 // Cast the predicate from svbool_t to the right number of elements.
3570 Ops[0] = EmitSVEPredicateCast(Pred: Ops[0], VTy: OverloadedTy);
3571
3572 // vector + imm addressing modes
3573 if (Ops[1]->getType()->isVectorTy()) {
3574 if (Ops.size() == 3) {
3575 // Pass 0 for 'vector+imm' when the index is omitted.
3576 Ops.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: 0));
3577
3578 // The sv_prfop is the last operand in the builtin and IR intrinsic.
3579 std::swap(a&: Ops[2], b&: Ops[3]);
3580 } else {
3581 // Index needs to be passed as scaled offset.
3582 llvm::Type *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
3583 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
3584 if (BytesPerElt > 1)
3585 Ops[2] = Builder.CreateShl(LHS: Ops[2], RHS: Log2_32(Value: BytesPerElt));
3586 }
3587
3588 Function *F = CGM.getIntrinsic(IID: IntID, Tys: OverloadedTy);
3589 return Builder.CreateCall(Callee: F, Args: Ops);
3590 }
3591
3592 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {Ops[1]->getType(), OverloadedTy});
3593 return Builder.CreateCall(Callee: F, Args: Ops);
3594}
3595
3596Value *CodeGenFunction::EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
3597 SmallVectorImpl<Value*> &Ops,
3598 unsigned IntID) {
3599 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
3600 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy);
3601 Value *BasePtr = Ops[1];
3602
3603 // Does the load have an offset?
3604 if (Ops.size() > 2)
3605 BasePtr = Builder.CreateGEP(Ty: VTy, Ptr: BasePtr, IdxList: Ops[2]);
3606
3607 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {VTy, BasePtr->getType()});
3608 return Builder.CreateCall(Callee: F, Args: {Predicate, BasePtr});
3609}
3610
3611Value *CodeGenFunction::EmitSVEStructStore(const SVETypeFlags &TypeFlags,
3612 SmallVectorImpl<Value*> &Ops,
3613 unsigned IntID) {
3614 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
3615
3616 unsigned N;
3617 switch (IntID) {
3618 case Intrinsic::aarch64_sve_st2:
3619 case Intrinsic::aarch64_sve_st1_pn_x2:
3620 case Intrinsic::aarch64_sve_stnt1_pn_x2:
3621 case Intrinsic::aarch64_sve_st2q:
3622 N = 2;
3623 break;
3624 case Intrinsic::aarch64_sve_st3:
3625 case Intrinsic::aarch64_sve_st3q:
3626 N = 3;
3627 break;
3628 case Intrinsic::aarch64_sve_st4:
3629 case Intrinsic::aarch64_sve_st1_pn_x4:
3630 case Intrinsic::aarch64_sve_stnt1_pn_x4:
3631 case Intrinsic::aarch64_sve_st4q:
3632 N = 4;
3633 break;
3634 default:
3635 llvm_unreachable("unknown intrinsic!");
3636 }
3637
3638 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy);
3639 Value *BasePtr = Ops[1];
3640
3641 // Does the store have an offset?
3642 if (Ops.size() > (2 + N))
3643 BasePtr = Builder.CreateGEP(Ty: VTy, Ptr: BasePtr, IdxList: Ops[2]);
3644
3645 // The llvm.aarch64.sve.st2/3/4 intrinsics take legal part vectors, so we
3646 // need to break up the tuple vector.
3647 SmallVector<llvm::Value*, 5> Operands;
3648 for (unsigned I = Ops.size() - N; I < Ops.size(); ++I)
3649 Operands.push_back(Elt: Ops[I]);
3650 Operands.append(IL: {Predicate, BasePtr});
3651 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {VTy, BasePtr->getType()});
3652
3653 return Builder.CreateCall(Callee: F, Args: Operands);
3654}
3655
3656// SVE2's svpmullb and svpmullt builtins are similar to the svpmullb_pair and
3657// svpmullt_pair intrinsics, with the exception that their results are bitcast
3658// to a wider type.
3659Value *CodeGenFunction::EmitSVEPMull(const SVETypeFlags &TypeFlags,
3660 SmallVectorImpl<Value *> &Ops,
3661 unsigned BuiltinID) {
3662 // Splat scalar operand to vector (intrinsics with _n infix)
3663 if (TypeFlags.hasSplatOperand()) {
3664 unsigned OpNo = TypeFlags.getSplatOperand();
3665 Ops[OpNo] = EmitSVEDupX(Scalar: Ops[OpNo]);
3666 }
3667
3668 // The pair-wise function has a narrower overloaded type.
3669 Function *F = CGM.getIntrinsic(IID: BuiltinID, Tys: Ops[0]->getType());
3670 Value *Call = Builder.CreateCall(Callee: F, Args: {Ops[0], Ops[1]});
3671
3672 // Now bitcast to the wider result type.
3673 llvm::ScalableVectorType *Ty = getSVEType(TypeFlags);
3674 return EmitSVEReinterpret(Val: Call, Ty);
3675}
3676
3677Value *CodeGenFunction::EmitSVEMovl(const SVETypeFlags &TypeFlags,
3678 ArrayRef<Value *> Ops, unsigned BuiltinID) {
3679 llvm::Type *OverloadedTy = getSVEType(TypeFlags);
3680 Function *F = CGM.getIntrinsic(IID: BuiltinID, Tys: OverloadedTy);
3681 return Builder.CreateCall(Callee: F, Args: {Ops[0], Builder.getInt32(C: 0)});
3682}
3683
3684Value *CodeGenFunction::EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
3685 SmallVectorImpl<Value *> &Ops,
3686 unsigned BuiltinID) {
3687 auto *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
3688 auto *VectorTy = getSVEVectorForElementType(EltTy: MemEltTy);
3689 auto *MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, SVTy: VectorTy);
3690
3691 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy: MemoryTy);
3692 Value *BasePtr = Ops[1];
3693
3694 // Implement the index operand if not omitted.
3695 if (Ops.size() > 3)
3696 BasePtr = Builder.CreateGEP(Ty: MemoryTy, Ptr: BasePtr, IdxList: Ops[2]);
3697
3698 Value *PrfOp = Ops.back();
3699
3700 llvm::Type *Tys[2] = {Predicate->getType(), BasePtr->getType()};
3701 Function *F = CGM.getIntrinsic(IID: BuiltinID, Tys);
3702 return Builder.CreateCall(Callee: F, Args: {Predicate, BasePtr, PrfOp});
3703}
3704
3705Value *CodeGenFunction::EmitSVEMaskedLoad(const CallExpr *E,
3706 llvm::Type *ReturnTy,
3707 SmallVectorImpl<Value *> &Ops,
3708 unsigned IntrinsicID,
3709 bool IsZExtReturn) {
3710 QualType LangPTy = E->getArg(Arg: 1)->getType();
3711 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
3712 T: LangPTy->castAs<PointerType>()->getPointeeType());
3713
3714 // Mfloat8 types is stored as a vector, so extra work
3715 // to extract sclar element type is necessary.
3716 if (MemEltTy->isVectorTy()) {
3717 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
3718 "Only <1 x i8> expected");
3719 MemEltTy = cast<llvm::VectorType>(Val: MemEltTy)->getElementType();
3720 }
3721
3722 // The vector type that is returned may be different from the
3723 // eventual type loaded from memory.
3724 auto VectorTy = cast<llvm::ScalableVectorType>(Val: ReturnTy);
3725 llvm::ScalableVectorType *MemoryTy = nullptr;
3726 llvm::ScalableVectorType *PredTy = nullptr;
3727 bool IsQuadLoad = false;
3728 switch (IntrinsicID) {
3729 case Intrinsic::aarch64_sve_ld1uwq:
3730 case Intrinsic::aarch64_sve_ld1udq:
3731 MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, MinNumElts: 1);
3732 PredTy = llvm::ScalableVectorType::get(
3733 ElementType: llvm::Type::getInt1Ty(C&: getLLVMContext()), MinNumElts: 1);
3734 IsQuadLoad = true;
3735 break;
3736 default:
3737 MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, SVTy: VectorTy);
3738 PredTy = MemoryTy;
3739 break;
3740 }
3741
3742 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy: PredTy);
3743 Value *BasePtr = Ops[1];
3744
3745 // Does the load have an offset?
3746 if (Ops.size() > 2)
3747 BasePtr = Builder.CreateGEP(Ty: MemoryTy, Ptr: BasePtr, IdxList: Ops[2]);
3748
3749 llvm::Type *Tys[2] = {IsQuadLoad ? VectorTy : MemoryTy, BasePtr->getType()};
3750 Function *F = CGM.getIntrinsic(IID: IntrinsicID, Tys);
3751 auto *Load = Builder.CreateCall(Callee: F, Args: {Predicate, BasePtr});
3752 auto TBAAInfo = CGM.getTBAAAccessInfo(AccessType: LangPTy->getPointeeType());
3753 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo);
3754
3755 if (IsQuadLoad)
3756 return Load;
3757
3758 return IsZExtReturn ? Builder.CreateZExt(V: Load, DestTy: VectorTy)
3759 : Builder.CreateSExt(V: Load, DestTy: VectorTy);
3760}
3761
3762Value *CodeGenFunction::EmitSVEMaskedStore(const CallExpr *E,
3763 SmallVectorImpl<Value *> &Ops,
3764 unsigned IntrinsicID) {
3765 QualType LangPTy = E->getArg(Arg: 1)->getType();
3766 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
3767 T: LangPTy->castAs<PointerType>()->getPointeeType());
3768
3769 // Mfloat8 types is stored as a vector, so extra work
3770 // to extract sclar element type is necessary.
3771 if (MemEltTy->isVectorTy()) {
3772 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
3773 "Only <1 x i8> expected");
3774 MemEltTy = cast<llvm::VectorType>(Val: MemEltTy)->getElementType();
3775 }
3776
3777 // The vector type that is stored may be different from the
3778 // eventual type stored to memory.
3779 auto VectorTy = cast<llvm::ScalableVectorType>(Val: Ops.back()->getType());
3780 auto MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, SVTy: VectorTy);
3781
3782 auto PredTy = MemoryTy;
3783 auto AddrMemoryTy = MemoryTy;
3784 bool IsQuadStore = false;
3785
3786 switch (IntrinsicID) {
3787 case Intrinsic::aarch64_sve_st1wq:
3788 case Intrinsic::aarch64_sve_st1dq:
3789 AddrMemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, MinNumElts: 1);
3790 PredTy =
3791 llvm::ScalableVectorType::get(ElementType: IntegerType::get(C&: getLLVMContext(), NumBits: 1), MinNumElts: 1);
3792 IsQuadStore = true;
3793 break;
3794 default:
3795 break;
3796 }
3797 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy: PredTy);
3798 Value *BasePtr = Ops[1];
3799
3800 // Does the store have an offset?
3801 if (Ops.size() == 4)
3802 BasePtr = Builder.CreateGEP(Ty: AddrMemoryTy, Ptr: BasePtr, IdxList: Ops[2]);
3803
3804 // Last value is always the data
3805 Value *Val =
3806 IsQuadStore ? Ops.back() : Builder.CreateTrunc(V: Ops.back(), DestTy: MemoryTy);
3807
3808 llvm::Type *Tys[2] = {IsQuadStore ? VectorTy : MemoryTy, BasePtr->getType()};
3809 Function *F = CGM.getIntrinsic(IID: IntrinsicID, Tys);
3810 auto *Store = Builder.CreateCall(Callee: F, Args: {Val, Predicate, BasePtr});
3811 auto TBAAInfo = CGM.getTBAAAccessInfo(AccessType: LangPTy->getPointeeType());
3812 CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
3813 return Store;
3814}
3815
3816Value *CodeGenFunction::EmitSMELd1St1(const SVETypeFlags &TypeFlags,
3817 SmallVectorImpl<Value *> &Ops,
3818 unsigned IntID) {
3819 Ops[2] = EmitSVEPredicateCast(
3820 Pred: Ops[2], VTy: getSVEVectorForElementType(EltTy: SVEBuiltinMemEltTy(TypeFlags)));
3821
3822 SmallVector<Value *> NewOps;
3823 NewOps.push_back(Elt: Ops[2]);
3824
3825 llvm::Value *BasePtr = Ops[3];
3826 llvm::Value *RealSlice = Ops[1];
3827 // If the intrinsic contains the vnum parameter, multiply it with the vector
3828 // size in bytes.
3829 if (Ops.size() == 5) {
3830 Function *StreamingVectorLength =
3831 CGM.getIntrinsic(IID: Intrinsic::aarch64_sme_cntsd);
3832 llvm::Value *StreamingVectorLengthCall =
3833 Builder.CreateMul(LHS: Builder.CreateCall(Callee: StreamingVectorLength),
3834 RHS: llvm::ConstantInt::get(Ty: Int64Ty, V: 8), Name: "svl",
3835 /* HasNUW */ true, /* HasNSW */ true);
3836 llvm::Value *Mulvl =
3837 Builder.CreateMul(LHS: StreamingVectorLengthCall, RHS: Ops[4], Name: "mulvl");
3838 // The type of the ptr parameter is void *, so use Int8Ty here.
3839 BasePtr = Builder.CreateGEP(Ty: Int8Ty, Ptr: Ops[3], IdxList: Mulvl);
3840 RealSlice = Builder.CreateZExt(V: RealSlice, DestTy: Int64Ty);
3841 RealSlice = Builder.CreateAdd(LHS: RealSlice, RHS: Ops[4]);
3842 RealSlice = Builder.CreateTrunc(V: RealSlice, DestTy: Int32Ty);
3843 }
3844 NewOps.push_back(Elt: BasePtr);
3845 NewOps.push_back(Elt: Ops[0]);
3846 NewOps.push_back(Elt: RealSlice);
3847 Function *F = CGM.getIntrinsic(IID: IntID, Tys: BasePtr->getType());
3848 return Builder.CreateCall(Callee: F, Args: NewOps);
3849}
3850
3851Value *CodeGenFunction::EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
3852 SmallVectorImpl<Value *> &Ops,
3853 unsigned IntID) {
3854 auto *VecTy = getSVEType(TypeFlags);
3855 Function *F = CGM.getIntrinsic(IID: IntID, Tys: VecTy);
3856 if (TypeFlags.isReadZA())
3857 Ops[1] = EmitSVEPredicateCast(Pred: Ops[1], VTy: VecTy);
3858 else if (TypeFlags.isWriteZA())
3859 Ops[2] = EmitSVEPredicateCast(Pred: Ops[2], VTy: VecTy);
3860 return Builder.CreateCall(Callee: F, Args: Ops);
3861}
3862
3863Value *CodeGenFunction::EmitSMEZero(const SVETypeFlags &TypeFlags,
3864 SmallVectorImpl<Value *> &Ops,
3865 unsigned IntID) {
3866 // svzero_za() intrinsic zeros the entire za tile and has no paramters.
3867 if (Ops.size() == 0)
3868 Ops.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: 255));
3869 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {});
3870 return Builder.CreateCall(Callee: F, Args: Ops);
3871}
3872
3873Value *CodeGenFunction::EmitSMELdrStr(const SVETypeFlags &TypeFlags,
3874 SmallVectorImpl<Value *> &Ops,
3875 unsigned IntID) {
3876 if (Ops.size() == 2)
3877 Ops.push_back(Elt: Builder.getInt32(C: 0));
3878 else
3879 Ops[2] = Builder.CreateIntCast(V: Ops[2], DestTy: Int32Ty, isSigned: true);
3880 Function *F = CGM.getIntrinsic(IID: IntID, Tys: Ops[1]->getType());
3881 return Builder.CreateCall(Callee: F, Args: Ops);
3882}
3883
3884// Limit the usage of scalable llvm IR generated by the ACLE by using the
3885// sve dup.x intrinsic instead of IRBuilder::CreateVectorSplat.
3886Value *CodeGenFunction::EmitSVEDupX(Value *Scalar, llvm::Type *Ty) {
3887 return Builder.CreateVectorSplat(
3888 EC: cast<llvm::VectorType>(Val: Ty)->getElementCount(), V: Scalar);
3889}
3890
3891Value *CodeGenFunction::EmitSVEDupX(Value *Scalar) {
3892 if (auto *Ty = Scalar->getType(); Ty->isVectorTy()) {
3893#ifndef NDEBUG
3894 auto *VecTy = cast<llvm::VectorType>(Ty);
3895 ElementCount EC = VecTy->getElementCount();
3896 assert(EC.isScalar() && VecTy->getElementType() == Int8Ty &&
3897 "Only <1 x i8> expected");
3898#endif
3899 Scalar = Builder.CreateExtractElement(Vec: Scalar, Idx: uint64_t(0));
3900 }
3901 return EmitSVEDupX(Scalar, Ty: getSVEVectorForElementType(EltTy: Scalar->getType()));
3902}
3903
3904Value *CodeGenFunction::EmitSVEReinterpret(Value *Val, llvm::Type *Ty) {
3905 // FIXME: For big endian this needs an additional REV, or needs a separate
3906 // intrinsic that is code-generated as a no-op, because the LLVM bitcast
3907 // instruction is defined as 'bitwise' equivalent from memory point of
3908 // view (when storing/reloading), whereas the svreinterpret builtin
3909 // implements bitwise equivalent cast from register point of view.
3910 // LLVM CodeGen for a bitcast must add an explicit REV for big-endian.
3911
3912 if (auto *StructTy = dyn_cast<StructType>(Val: Ty)) {
3913 Value *Tuple = llvm::PoisonValue::get(T: Ty);
3914
3915 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
3916 Value *In = Builder.CreateExtractValue(Agg: Val, Idxs: I);
3917 Value *Out = Builder.CreateBitCast(V: In, DestTy: StructTy->getTypeAtIndex(N: I));
3918 Tuple = Builder.CreateInsertValue(Agg: Tuple, Val: Out, Idxs: I);
3919 }
3920
3921 return Tuple;
3922 }
3923
3924 return Builder.CreateBitCast(V: Val, DestTy: Ty);
3925}
3926
3927static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty,
3928 SmallVectorImpl<Value *> &Ops) {
3929 auto *SplatZero = Constant::getNullValue(Ty);
3930 Ops.insert(I: Ops.begin(), Elt: SplatZero);
3931}
3932
3933static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty,
3934 SmallVectorImpl<Value *> &Ops) {
3935 auto *SplatUndef = UndefValue::get(T: Ty);
3936 Ops.insert(I: Ops.begin(), Elt: SplatUndef);
3937}
3938
3939SmallVector<llvm::Type *, 2>
3940CodeGenFunction::getSVEOverloadTypes(const SVETypeFlags &TypeFlags,
3941 llvm::Type *ResultType,
3942 ArrayRef<Value *> Ops) {
3943 if (TypeFlags.isOverloadNone())
3944 return {};
3945
3946 llvm::Type *DefaultType = getSVEType(TypeFlags);
3947
3948 if (TypeFlags.isOverloadWhileOrMultiVecCvt())
3949 return {DefaultType, Ops[1]->getType()};
3950
3951 if (TypeFlags.isOverloadWhileRW())
3952 return {getSVEPredType(TypeFlags), Ops[0]->getType()};
3953
3954 if (TypeFlags.isOverloadDefaultAndOp0())
3955 return {DefaultType, Ops[0]->getType()};
3956
3957 if (TypeFlags.isOverloadFirstandLast())
3958 return {Ops[0]->getType(), Ops.back()->getType()};
3959
3960 if (TypeFlags.isReductionQV())
3961 return {ResultType, Ops[1]->getType()};
3962
3963 assert(TypeFlags.isOverloadDefault() && "Unexpected value for overloads");
3964 return {DefaultType};
3965}
3966
3967Value *CodeGenFunction::EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
3968 ArrayRef<Value *> Ops) {
3969 assert((TypeFlags.isTupleSet() || TypeFlags.isTupleGet()) &&
3970 "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()");
3971 unsigned Idx = cast<ConstantInt>(Val: Ops[1])->getZExtValue();
3972
3973 if (TypeFlags.isTupleSet())
3974 return Builder.CreateInsertValue(Agg: Ops[0], Val: Ops[2], Idxs: Idx);
3975 return Builder.CreateExtractValue(Agg: Ops[0], Idxs: Idx);
3976}
3977
3978Value *CodeGenFunction::EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
3979 llvm::Type *Ty,
3980 ArrayRef<Value *> Ops) {
3981 assert(TypeFlags.isTupleCreate() && "Expects TypleFlag isTupleCreate");
3982
3983 Value *Tuple = llvm::PoisonValue::get(T: Ty);
3984 for (unsigned Idx = 0; Idx < Ops.size(); Idx++)
3985 Tuple = Builder.CreateInsertValue(Agg: Tuple, Val: Ops[Idx], Idxs: Idx);
3986
3987 return Tuple;
3988}
3989
3990void CodeGenFunction::GetAArch64SVEProcessedOperands(
3991 unsigned BuiltinID, const CallExpr *E, SmallVectorImpl<Value *> &Ops,
3992 SVETypeFlags TypeFlags) {
3993 // Find out if any arguments are required to be integer constant expressions.
3994 unsigned ICEArguments = 0;
3995 ASTContext::GetBuiltinTypeError Error;
3996 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
3997 assert(Error == ASTContext::GE_None && "Should not codegen an error");
3998
3999 // Tuple set/get only requires one insert/extract vector, which is
4000 // created by EmitSVETupleSetOrGet.
4001 bool IsTupleGetOrSet = TypeFlags.isTupleSet() || TypeFlags.isTupleGet();
4002
4003 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
4004 bool IsICE = ICEArguments & (1 << i);
4005 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: i));
4006
4007 if (IsICE) {
4008 // If this is required to be a constant, constant fold it so that we know
4009 // that the generated intrinsic gets a ConstantInt.
4010 std::optional<llvm::APSInt> Result =
4011 E->getArg(Arg: i)->getIntegerConstantExpr(Ctx: getContext());
4012 assert(Result && "Expected argument to be a constant");
4013
4014 // Immediates for SVE llvm intrinsics are always 32bit. We can safely
4015 // truncate because the immediate has been range checked and no valid
4016 // immediate requires more than a handful of bits.
4017 *Result = Result->extOrTrunc(width: 32);
4018 Ops.push_back(Elt: llvm::ConstantInt::get(Context&: getLLVMContext(), V: *Result));
4019 continue;
4020 }
4021
4022 if (isa<StructType>(Val: Arg->getType()) && !IsTupleGetOrSet) {
4023 for (unsigned I = 0; I < Arg->getType()->getStructNumElements(); ++I)
4024 Ops.push_back(Elt: Builder.CreateExtractValue(Agg: Arg, Idxs: I));
4025
4026 continue;
4027 }
4028
4029 Ops.push_back(Elt: Arg);
4030 }
4031}
4032
4033Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
4034 const CallExpr *E) {
4035 llvm::Type *Ty = ConvertType(T: E->getType());
4036 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
4037 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
4038 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
4039 return EmitSVEReinterpret(Val, Ty);
4040 }
4041
4042 auto *Builtin =
4043 findARMVectorIntrinsicInMap(IntrinsicMap: ArrayRef(AArch64SVEIntrinsicMap), BuiltinID,
4044 MapProvenSorted&: AArch64SVEIntrinsicsProvenSorted);
4045
4046 llvm::SmallVector<Value *, 4> Ops;
4047 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4048 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
4049
4050 if (TypeFlags.isLoad())
4051 return EmitSVEMaskedLoad(E, ReturnTy: Ty, Ops, IntrinsicID: Builtin->LLVMIntrinsic,
4052 IsZExtReturn: TypeFlags.isZExtReturn());
4053 if (TypeFlags.isStore())
4054 return EmitSVEMaskedStore(E, Ops, IntrinsicID: Builtin->LLVMIntrinsic);
4055 if (TypeFlags.isGatherLoad())
4056 return EmitSVEGatherLoad(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4057 if (TypeFlags.isScatterStore())
4058 return EmitSVEScatterStore(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4059 if (TypeFlags.isPrefetch())
4060 return EmitSVEPrefetchLoad(TypeFlags, Ops, BuiltinID: Builtin->LLVMIntrinsic);
4061 if (TypeFlags.isGatherPrefetch())
4062 return EmitSVEGatherPrefetch(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4063 if (TypeFlags.isStructLoad())
4064 return EmitSVEStructLoad(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4065 if (TypeFlags.isStructStore())
4066 return EmitSVEStructStore(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4067 if (TypeFlags.isTupleSet() || TypeFlags.isTupleGet())
4068 return EmitSVETupleSetOrGet(TypeFlags, Ops);
4069 if (TypeFlags.isTupleCreate())
4070 return EmitSVETupleCreate(TypeFlags, Ty, Ops);
4071 if (TypeFlags.isUndef())
4072 return UndefValue::get(T: Ty);
4073
4074 // Handle built-ins for which there is a corresponding LLVM Intrinsic.
4075 // -------------------------------------------------------------------
4076 if (Builtin->LLVMIntrinsic != 0) {
4077 // Emit set FPMR for intrinsics that require it
4078 if (TypeFlags.setsFPMR())
4079 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_set_fpmr),
4080 Args: Ops.pop_back_val());
4081 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZeroExp)
4082 InsertExplicitZeroOperand(Builder, Ty, Ops);
4083
4084 if (TypeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
4085 InsertExplicitUndefOperand(Builder, Ty, Ops);
4086
4087 // Some ACLE builtins leave out the argument to specify the predicate
4088 // pattern, which is expected to be expanded to an SV_ALL pattern.
4089 if (TypeFlags.isAppendSVALL())
4090 Ops.push_back(Elt: Builder.getInt32(/*SV_ALL*/ C: 31));
4091 if (TypeFlags.isInsertOp1SVALL())
4092 Ops.insert(I: &Ops[1], Elt: Builder.getInt32(/*SV_ALL*/ C: 31));
4093
4094 // Predicates must match the main datatype.
4095 for (Value *&Op : Ops)
4096 if (auto PredTy = dyn_cast<llvm::VectorType>(Val: Op->getType()))
4097 if (PredTy->getElementType()->isIntegerTy(BitWidth: 1))
4098 Op = EmitSVEPredicateCast(Pred: Op, VTy: getSVEType(TypeFlags));
4099
4100 // Splat scalar operand to vector (intrinsics with _n infix)
4101 if (TypeFlags.hasSplatOperand()) {
4102 unsigned OpNo = TypeFlags.getSplatOperand();
4103 Ops[OpNo] = EmitSVEDupX(Scalar: Ops[OpNo]);
4104 }
4105
4106 if (TypeFlags.isReverseCompare())
4107 std::swap(a&: Ops[1], b&: Ops[2]);
4108 else if (TypeFlags.isReverseUSDOT())
4109 std::swap(a&: Ops[1], b&: Ops[2]);
4110 else if (TypeFlags.isReverseMergeAnyBinOp() &&
4111 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4112 std::swap(a&: Ops[1], b&: Ops[2]);
4113 else if (TypeFlags.isReverseMergeAnyAccOp() &&
4114 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4115 std::swap(a&: Ops[1], b&: Ops[3]);
4116
4117 // Predicated intrinsics with _z suffix need a select w/ zeroinitializer.
4118 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZero) {
4119 llvm::Type *OpndTy = Ops[1]->getType();
4120 auto *SplatZero = Constant::getNullValue(Ty: OpndTy);
4121 Ops[1] = Builder.CreateSelect(C: Ops[0], True: Ops[1], False: SplatZero);
4122 }
4123
4124 Function *F = CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic,
4125 Tys: getSVEOverloadTypes(TypeFlags, ResultType: Ty, Ops));
4126 Value *Call = Builder.CreateCall(Callee: F, Args: Ops);
4127
4128 if (Call->getType() == Ty)
4129 return Call;
4130
4131 // Predicate results must be converted to svbool_t.
4132 if (auto PredTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty))
4133 return EmitSVEPredicateCast(Pred: Call, VTy: PredTy);
4134 if (auto PredTupleTy = dyn_cast<llvm::StructType>(Val: Ty))
4135 return EmitSVEPredicateTupleCast(PredTuple: Call, Ty: PredTupleTy);
4136
4137 llvm_unreachable("unsupported element count!");
4138 }
4139
4140 switch (BuiltinID) {
4141 default:
4142 return nullptr;
4143
4144 case SVE::BI__builtin_sve_svreinterpret_b: {
4145 Function *CastFromSVCountF =
4146 CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_convert_from_svcount);
4147 return Builder.CreateCall(Callee: CastFromSVCountF, Args: Ops[0]);
4148 }
4149 case SVE::BI__builtin_sve_svreinterpret_c: {
4150 Function *CastToSVCountF =
4151 CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_convert_to_svcount);
4152 return Builder.CreateCall(Callee: CastToSVCountF, Args: Ops[0]);
4153 }
4154
4155 case SVE::BI__builtin_sve_svpsel_lane_b8:
4156 case SVE::BI__builtin_sve_svpsel_lane_b16:
4157 case SVE::BI__builtin_sve_svpsel_lane_b32:
4158 case SVE::BI__builtin_sve_svpsel_lane_b64:
4159 case SVE::BI__builtin_sve_svpsel_lane_c8:
4160 case SVE::BI__builtin_sve_svpsel_lane_c16:
4161 case SVE::BI__builtin_sve_svpsel_lane_c32:
4162 case SVE::BI__builtin_sve_svpsel_lane_c64: {
4163 auto OverloadedTy = getSVEType(TypeFlags: SVETypeFlags(Builtin->TypeModifier));
4164 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_psel,
4165 Tys: {Ops[0]->getType(), OverloadedTy});
4166 llvm::Value *Ops1 = EmitSVEPredicateCast(Pred: Ops[1], VTy: OverloadedTy);
4167 return Builder.CreateCall(Callee: F, Args: {Ops[0], Ops1, Ops[2]});
4168 }
4169 case SVE::BI__builtin_sve_svmov_b_z: {
4170 // svmov_b_z(pg, op) <=> svand_b_z(pg, op, op)
4171 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4172 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4173 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_and_z, Tys: OverloadedTy);
4174 return Builder.CreateCall(Callee: F, Args: {Ops[0], Ops[1], Ops[1]});
4175 }
4176
4177 case SVE::BI__builtin_sve_svnot_b_z: {
4178 // svnot_b_z(pg, op) <=> sveor_b_z(pg, op, pg)
4179 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4180 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4181 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_eor_z, Tys: OverloadedTy);
4182 return Builder.CreateCall(Callee: F, Args: {Ops[0], Ops[1], Ops[0]});
4183 }
4184
4185 case SVE::BI__builtin_sve_svmovlb_u16:
4186 case SVE::BI__builtin_sve_svmovlb_u32:
4187 case SVE::BI__builtin_sve_svmovlb_u64:
4188 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_ushllb);
4189
4190 case SVE::BI__builtin_sve_svmovlb_s16:
4191 case SVE::BI__builtin_sve_svmovlb_s32:
4192 case SVE::BI__builtin_sve_svmovlb_s64:
4193 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_sshllb);
4194
4195 case SVE::BI__builtin_sve_svmovlt_u16:
4196 case SVE::BI__builtin_sve_svmovlt_u32:
4197 case SVE::BI__builtin_sve_svmovlt_u64:
4198 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_ushllt);
4199
4200 case SVE::BI__builtin_sve_svmovlt_s16:
4201 case SVE::BI__builtin_sve_svmovlt_s32:
4202 case SVE::BI__builtin_sve_svmovlt_s64:
4203 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_sshllt);
4204
4205 case SVE::BI__builtin_sve_svpmullt_u16:
4206 case SVE::BI__builtin_sve_svpmullt_u64:
4207 case SVE::BI__builtin_sve_svpmullt_n_u16:
4208 case SVE::BI__builtin_sve_svpmullt_n_u64:
4209 return EmitSVEPMull(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_pmullt_pair);
4210
4211 case SVE::BI__builtin_sve_svpmullb_u16:
4212 case SVE::BI__builtin_sve_svpmullb_u64:
4213 case SVE::BI__builtin_sve_svpmullb_n_u16:
4214 case SVE::BI__builtin_sve_svpmullb_n_u64:
4215 return EmitSVEPMull(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_pmullb_pair);
4216
4217 case SVE::BI__builtin_sve_svdup_n_b8:
4218 case SVE::BI__builtin_sve_svdup_n_b16:
4219 case SVE::BI__builtin_sve_svdup_n_b32:
4220 case SVE::BI__builtin_sve_svdup_n_b64: {
4221 llvm::ScalableVectorType *OverloadedTy = getSVEType(TypeFlags);
4222 Value *Dup = EmitSVEDupX(Scalar: Ops[0], Ty: OverloadedTy);
4223 return EmitSVEPredicateCast(Pred: Dup, VTy: cast<llvm::ScalableVectorType>(Val: Ty));
4224 }
4225
4226 case SVE::BI__builtin_sve_svdupq_n_b8:
4227 case SVE::BI__builtin_sve_svdupq_n_b16:
4228 case SVE::BI__builtin_sve_svdupq_n_b32:
4229 case SVE::BI__builtin_sve_svdupq_n_b64:
4230 case SVE::BI__builtin_sve_svdupq_n_u8:
4231 case SVE::BI__builtin_sve_svdupq_n_s8:
4232 case SVE::BI__builtin_sve_svdupq_n_u64:
4233 case SVE::BI__builtin_sve_svdupq_n_f64:
4234 case SVE::BI__builtin_sve_svdupq_n_s64:
4235 case SVE::BI__builtin_sve_svdupq_n_u16:
4236 case SVE::BI__builtin_sve_svdupq_n_f16:
4237 case SVE::BI__builtin_sve_svdupq_n_bf16:
4238 case SVE::BI__builtin_sve_svdupq_n_s16:
4239 case SVE::BI__builtin_sve_svdupq_n_u32:
4240 case SVE::BI__builtin_sve_svdupq_n_f32:
4241 case SVE::BI__builtin_sve_svdupq_n_s32: {
4242 // These builtins are implemented by storing each element to an array and using
4243 // ld1rq to materialize a vector.
4244 unsigned NumOpnds = Ops.size();
4245
4246 bool IsBoolTy =
4247 cast<llvm::VectorType>(Val: Ty)->getElementType()->isIntegerTy(BitWidth: 1);
4248
4249 // For svdupq_n_b* the element type of is an integer of type 128/numelts,
4250 // so that the compare can use the width that is natural for the expected
4251 // number of predicate lanes.
4252 llvm::Type *EltTy = Ops[0]->getType();
4253 if (IsBoolTy)
4254 EltTy = IntegerType::get(C&: getLLVMContext(), NumBits: SVEBitsPerBlock / NumOpnds);
4255
4256 SmallVector<llvm::Value *, 16> VecOps;
4257 for (unsigned I = 0; I < NumOpnds; ++I)
4258 VecOps.push_back(Elt: Builder.CreateZExt(V: Ops[I], DestTy: EltTy));
4259 Value *Vec = BuildVector(Ops: VecOps);
4260
4261 llvm::Type *OverloadedTy = getSVEVectorForElementType(EltTy);
4262 Value *InsertSubVec = Builder.CreateInsertVector(
4263 DstType: OverloadedTy, SrcVec: PoisonValue::get(T: OverloadedTy), SubVec: Vec, Idx: uint64_t(0));
4264
4265 Function *F =
4266 CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_dupq_lane, Tys: OverloadedTy);
4267 Value *DupQLane =
4268 Builder.CreateCall(Callee: F, Args: {InsertSubVec, Builder.getInt64(C: 0)});
4269
4270 if (!IsBoolTy)
4271 return DupQLane;
4272
4273 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4274 Constant *Pred = ConstantInt::getTrue(Ty: getSVEPredType(TypeFlags));
4275
4276 // For svdupq_n_b* we need to add an additional 'cmpne' with '0'.
4277 F = CGM.getIntrinsic(IID: NumOpnds == 2 ? Intrinsic::aarch64_sve_cmpne
4278 : Intrinsic::aarch64_sve_cmpne_wide,
4279 Tys: OverloadedTy);
4280 Value *Call = Builder.CreateCall(
4281 Callee: F, Args: {Pred, DupQLane, EmitSVEDupX(Scalar: Builder.getInt64(C: 0))});
4282 return EmitSVEPredicateCast(Pred: Call, VTy: cast<llvm::ScalableVectorType>(Val: Ty));
4283 }
4284
4285 case SVE::BI__builtin_sve_svpfalse_b:
4286 return ConstantInt::getFalse(Ty);
4287
4288 case SVE::BI__builtin_sve_svpfalse_c:
4289 return Constant::getNullValue(Ty);
4290
4291 case SVE::BI__builtin_sve_svlen_bf16:
4292 case SVE::BI__builtin_sve_svlen_f16:
4293 case SVE::BI__builtin_sve_svlen_f32:
4294 case SVE::BI__builtin_sve_svlen_f64:
4295 case SVE::BI__builtin_sve_svlen_s8:
4296 case SVE::BI__builtin_sve_svlen_s16:
4297 case SVE::BI__builtin_sve_svlen_s32:
4298 case SVE::BI__builtin_sve_svlen_s64:
4299 case SVE::BI__builtin_sve_svlen_u8:
4300 case SVE::BI__builtin_sve_svlen_u16:
4301 case SVE::BI__builtin_sve_svlen_u32:
4302 case SVE::BI__builtin_sve_svlen_u64: {
4303 SVETypeFlags TF(Builtin->TypeModifier);
4304 return Builder.CreateElementCount(Ty, EC: getSVEType(TypeFlags: TF)->getElementCount());
4305 }
4306
4307 case SVE::BI__builtin_sve_svtbl2_u8:
4308 case SVE::BI__builtin_sve_svtbl2_s8:
4309 case SVE::BI__builtin_sve_svtbl2_u16:
4310 case SVE::BI__builtin_sve_svtbl2_s16:
4311 case SVE::BI__builtin_sve_svtbl2_u32:
4312 case SVE::BI__builtin_sve_svtbl2_s32:
4313 case SVE::BI__builtin_sve_svtbl2_u64:
4314 case SVE::BI__builtin_sve_svtbl2_s64:
4315 case SVE::BI__builtin_sve_svtbl2_f16:
4316 case SVE::BI__builtin_sve_svtbl2_bf16:
4317 case SVE::BI__builtin_sve_svtbl2_f32:
4318 case SVE::BI__builtin_sve_svtbl2_f64: {
4319 SVETypeFlags TF(Builtin->TypeModifier);
4320 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_tbl2, Tys: getSVEType(TypeFlags: TF));
4321 return Builder.CreateCall(Callee: F, Args: Ops);
4322 }
4323
4324 case SVE::BI__builtin_sve_svset_neonq_s8:
4325 case SVE::BI__builtin_sve_svset_neonq_s16:
4326 case SVE::BI__builtin_sve_svset_neonq_s32:
4327 case SVE::BI__builtin_sve_svset_neonq_s64:
4328 case SVE::BI__builtin_sve_svset_neonq_u8:
4329 case SVE::BI__builtin_sve_svset_neonq_u16:
4330 case SVE::BI__builtin_sve_svset_neonq_u32:
4331 case SVE::BI__builtin_sve_svset_neonq_u64:
4332 case SVE::BI__builtin_sve_svset_neonq_f16:
4333 case SVE::BI__builtin_sve_svset_neonq_f32:
4334 case SVE::BI__builtin_sve_svset_neonq_f64:
4335 case SVE::BI__builtin_sve_svset_neonq_bf16:
4336 case SVE::BI__builtin_sve_svset_neonq_mf8: {
4337 return Builder.CreateInsertVector(DstType: Ty, SrcVec: Ops[0], SubVec: Ops[1], Idx: uint64_t(0));
4338 }
4339
4340 case SVE::BI__builtin_sve_svget_neonq_s8:
4341 case SVE::BI__builtin_sve_svget_neonq_s16:
4342 case SVE::BI__builtin_sve_svget_neonq_s32:
4343 case SVE::BI__builtin_sve_svget_neonq_s64:
4344 case SVE::BI__builtin_sve_svget_neonq_u8:
4345 case SVE::BI__builtin_sve_svget_neonq_u16:
4346 case SVE::BI__builtin_sve_svget_neonq_u32:
4347 case SVE::BI__builtin_sve_svget_neonq_u64:
4348 case SVE::BI__builtin_sve_svget_neonq_f16:
4349 case SVE::BI__builtin_sve_svget_neonq_f32:
4350 case SVE::BI__builtin_sve_svget_neonq_f64:
4351 case SVE::BI__builtin_sve_svget_neonq_bf16:
4352 case SVE::BI__builtin_sve_svget_neonq_mf8: {
4353 return Builder.CreateExtractVector(DstType: Ty, SrcVec: Ops[0], Idx: uint64_t(0));
4354 }
4355
4356 case SVE::BI__builtin_sve_svdup_neonq_s8:
4357 case SVE::BI__builtin_sve_svdup_neonq_s16:
4358 case SVE::BI__builtin_sve_svdup_neonq_s32:
4359 case SVE::BI__builtin_sve_svdup_neonq_s64:
4360 case SVE::BI__builtin_sve_svdup_neonq_u8:
4361 case SVE::BI__builtin_sve_svdup_neonq_u16:
4362 case SVE::BI__builtin_sve_svdup_neonq_u32:
4363 case SVE::BI__builtin_sve_svdup_neonq_u64:
4364 case SVE::BI__builtin_sve_svdup_neonq_f16:
4365 case SVE::BI__builtin_sve_svdup_neonq_f32:
4366 case SVE::BI__builtin_sve_svdup_neonq_f64:
4367 case SVE::BI__builtin_sve_svdup_neonq_bf16:
4368 case SVE::BI__builtin_sve_svdup_neonq_mf8: {
4369 Value *Insert = Builder.CreateInsertVector(DstType: Ty, SrcVec: PoisonValue::get(T: Ty), SubVec: Ops[0],
4370 Idx: uint64_t(0));
4371 return Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_dupq_lane, OverloadTypes: {Ty},
4372 Args: {Insert, Builder.getInt64(C: 0)});
4373 }
4374 }
4375
4376 /// Should not happen
4377 return nullptr;
4378}
4379
4380static void swapCommutativeSMEOperands(unsigned BuiltinID,
4381 SmallVectorImpl<Value *> &Ops) {
4382 unsigned MultiVec;
4383 switch (BuiltinID) {
4384 default:
4385 return;
4386 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
4387 MultiVec = 1;
4388 break;
4389 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
4390 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
4391 MultiVec = 2;
4392 break;
4393 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
4394 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
4395 MultiVec = 4;
4396 break;
4397 }
4398
4399 if (MultiVec > 0)
4400 for (unsigned I = 0; I < MultiVec; ++I)
4401 std::swap(a&: Ops[I + 1], b&: Ops[I + 1 + MultiVec]);
4402}
4403
4404Value *CodeGenFunction::EmitAArch64SMEBuiltinExpr(unsigned BuiltinID,
4405 const CallExpr *E) {
4406 auto *Builtin =
4407 findARMVectorIntrinsicInMap(IntrinsicMap: ArrayRef(AArch64SMEIntrinsicMap), BuiltinID,
4408 MapProvenSorted&: AArch64SMEIntrinsicsProvenSorted);
4409
4410 llvm::SmallVector<Value *, 4> Ops;
4411 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4412 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
4413
4414 if (TypeFlags.isLoad() || TypeFlags.isStore())
4415 return EmitSMELd1St1(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4416 if (TypeFlags.isReadZA() || TypeFlags.isWriteZA())
4417 return EmitSMEReadWrite(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4418 if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
4419 BuiltinID == SME::BI__builtin_sme_svzero_za)
4420 return EmitSMEZero(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4421 if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
4422 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
4423 BuiltinID == SME::BI__builtin_sme_svldr_za ||
4424 BuiltinID == SME::BI__builtin_sme_svstr_za)
4425 return EmitSMELdrStr(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4426
4427 // Emit set FPMR for intrinsics that require it
4428 if (TypeFlags.setsFPMR())
4429 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_set_fpmr),
4430 Args: Ops.pop_back_val());
4431 // Handle builtins which require their multi-vector operands to be swapped
4432 swapCommutativeSMEOperands(BuiltinID, Ops);
4433
4434 auto isCntsBuiltin = [&]() {
4435 switch (BuiltinID) {
4436 default:
4437 return 0;
4438 case SME::BI__builtin_sme_svcntsb:
4439 return 8;
4440 case SME::BI__builtin_sme_svcntsh:
4441 return 4;
4442 case SME::BI__builtin_sme_svcntsw:
4443 return 2;
4444 }
4445 };
4446
4447 if (auto Mul = isCntsBuiltin()) {
4448 llvm::Value *Cntd =
4449 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_sme_cntsd));
4450 return Builder.CreateMul(LHS: Cntd, RHS: llvm::ConstantInt::get(Ty: Int64Ty, V: Mul),
4451 Name: "mulsvl", /* HasNUW */ true, /* HasNSW */ true);
4452 }
4453
4454 // Should not happen!
4455 if (Builtin->LLVMIntrinsic == 0)
4456 return nullptr;
4457
4458 // Predicates must match the main datatype.
4459 for (Value *&Op : Ops)
4460 if (auto PredTy = dyn_cast<llvm::VectorType>(Val: Op->getType()))
4461 if (PredTy->getElementType()->isIntegerTy(BitWidth: 1))
4462 Op = EmitSVEPredicateCast(Pred: Op, VTy: getSVEType(TypeFlags));
4463
4464 if (BuiltinID == SME::BI__builtin_sme_svldr_zt ||
4465 BuiltinID == SME::BI__builtin_sme_svstr_zt) {
4466 Function *F = CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic, Tys: Ops[1]->getType());
4467 return Builder.CreateCall(Callee: F, Args: Ops);
4468 }
4469
4470 Function *F =
4471 TypeFlags.isOverloadNone()
4472 ? CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic)
4473 : CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic, Tys: {getSVEType(TypeFlags)});
4474
4475 return Builder.CreateCall(Callee: F, Args: Ops);
4476}
4477
4478/// Helper for the read/write/add/inc X18 builtins: read the X18 register and
4479/// return it as an i8 pointer.
4480Value *readX18AsPtr(CodeGenFunction &CGF) {
4481 LLVMContext &Context = CGF.CGM.getLLVMContext();
4482 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Str: "x18")};
4483 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
4484 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
4485 llvm::Function *F =
4486 CGF.CGM.getIntrinsic(IID: Intrinsic::read_register, Tys: {CGF.Int64Ty});
4487 llvm::Value *X18 = CGF.Builder.CreateCall(Callee: F, Args: Metadata);
4488 return CGF.Builder.CreateIntToPtr(V: X18, DestTy: CGF.Int8PtrTy);
4489}
4490
4491Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
4492 const CallExpr *E,
4493 llvm::Triple::ArchType Arch) {
4494 if (BuiltinID >= clang::AArch64::FirstSVEBuiltin &&
4495 BuiltinID <= clang::AArch64::LastSVEBuiltin)
4496 return EmitAArch64SVEBuiltinExpr(BuiltinID, E);
4497
4498 if (BuiltinID >= clang::AArch64::FirstSMEBuiltin &&
4499 BuiltinID <= clang::AArch64::LastSMEBuiltin)
4500 return EmitAArch64SMEBuiltinExpr(BuiltinID, E);
4501
4502 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
4503 return EmitAArch64CpuSupports(E);
4504
4505 unsigned HintID = static_cast<unsigned>(-1);
4506 switch (BuiltinID) {
4507 default: break;
4508 case clang::AArch64::BI__builtin_arm_nop:
4509 HintID = 0;
4510 break;
4511 case clang::AArch64::BI__builtin_arm_yield:
4512 case clang::AArch64::BI__yield:
4513 HintID = 1;
4514 break;
4515 case clang::AArch64::BI__builtin_arm_wfe:
4516 case clang::AArch64::BI__wfe:
4517 HintID = 2;
4518 break;
4519 case clang::AArch64::BI__builtin_arm_wfi:
4520 case clang::AArch64::BI__wfi:
4521 HintID = 3;
4522 break;
4523 case clang::AArch64::BI__builtin_arm_sev:
4524 case clang::AArch64::BI__sev:
4525 HintID = 4;
4526 break;
4527 case clang::AArch64::BI__builtin_arm_sevl:
4528 case clang::AArch64::BI__sevl:
4529 HintID = 5;
4530 break;
4531 }
4532
4533 if (HintID != static_cast<unsigned>(-1)) {
4534 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_hint);
4535 return Builder.CreateCall(Callee: F, Args: llvm::ConstantInt::get(Ty: Int32Ty, V: HintID));
4536 }
4537
4538 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
4539 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_break);
4540 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4541 return Builder.CreateCall(Callee: F, Args: Builder.CreateZExt(V: Arg, DestTy: CGM.Int32Ty));
4542 }
4543
4544 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
4545 // Create call to __arm_sme_state and store the results to the two pointers.
4546 CallInst *CI = EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(
4547 Ty: llvm::FunctionType::get(Result: StructType::get(elt1: CGM.Int64Ty, elts: CGM.Int64Ty), Params: {},
4548 isVarArg: false),
4549 Name: "__arm_sme_state"));
4550 auto Attrs = AttributeList().addFnAttribute(C&: getLLVMContext(),
4551 Kind: "aarch64_pstate_sm_compatible");
4552 CI->setAttributes(Attrs);
4553 CI->setCallingConv(
4554 llvm::CallingConv::
4555 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
4556 Builder.CreateStore(Val: Builder.CreateExtractValue(Agg: CI, Idxs: 0),
4557 Addr: EmitPointerWithAlignment(Addr: E->getArg(Arg: 0)));
4558 return Builder.CreateStore(Val: Builder.CreateExtractValue(Agg: CI, Idxs: 1),
4559 Addr: EmitPointerWithAlignment(Addr: E->getArg(Arg: 1)));
4560 }
4561
4562 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
4563 assert((getContext().getTypeSize(E->getType()) == 32) &&
4564 "rbit of unusual size!");
4565 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4566 return Builder.CreateCall(
4567 Callee: CGM.getIntrinsic(IID: Intrinsic::bitreverse, Tys: Arg->getType()), Args: Arg, Name: "rbit");
4568 }
4569 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
4570 assert((getContext().getTypeSize(E->getType()) == 64) &&
4571 "rbit of unusual size!");
4572 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4573 return Builder.CreateCall(
4574 Callee: CGM.getIntrinsic(IID: Intrinsic::bitreverse, Tys: Arg->getType()), Args: Arg, Name: "rbit");
4575 }
4576
4577 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
4578 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
4579 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4580 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: Arg->getType());
4581 Value *Res = Builder.CreateCall(Callee: F, Args: {Arg, Builder.getInt1(V: false)});
4582 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
4583 Res = Builder.CreateTrunc(V: Res, DestTy: Builder.getInt32Ty());
4584 return Res;
4585 }
4586
4587 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
4588 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4589 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_cls), Args: Arg,
4590 Name: "cls");
4591 }
4592 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
4593 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4594 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_cls64), Args: Arg,
4595 Name: "cls");
4596 }
4597
4598 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
4599 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
4600 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4601 llvm::Type *Ty = Arg->getType();
4602 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint32z, Tys: Ty),
4603 Args: Arg, Name: "frint32z");
4604 }
4605
4606 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
4607 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
4608 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4609 llvm::Type *Ty = Arg->getType();
4610 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint64z, Tys: Ty),
4611 Args: Arg, Name: "frint64z");
4612 }
4613
4614 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
4615 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
4616 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4617 llvm::Type *Ty = Arg->getType();
4618 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint32x, Tys: Ty),
4619 Args: Arg, Name: "frint32x");
4620 }
4621
4622 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
4623 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
4624 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4625 llvm::Type *Ty = Arg->getType();
4626 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint64x, Tys: Ty),
4627 Args: Arg, Name: "frint64x");
4628 }
4629
4630 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
4631 assert((getContext().getTypeSize(E->getType()) == 32) &&
4632 "__jcvt of unusual size!");
4633 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4634 return Builder.CreateCall(
4635 Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_fjcvtzs), Args: Arg);
4636 }
4637
4638 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
4639 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
4640 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
4641 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
4642 llvm::Value *MemAddr = EmitScalarExpr(E: E->getArg(Arg: 0));
4643 llvm::Value *ValPtr = EmitScalarExpr(E: E->getArg(Arg: 1));
4644
4645 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
4646 // Load from the address via an LLVM intrinsic, receiving a
4647 // tuple of 8 i64 words, and store each one to ValPtr.
4648 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_ld64b);
4649 llvm::Value *Val = Builder.CreateCall(Callee: F, Args: MemAddr);
4650 llvm::Value *ToRet;
4651 for (size_t i = 0; i < 8; i++) {
4652 llvm::Value *ValOffsetPtr =
4653 Builder.CreateGEP(Ty: Int64Ty, Ptr: ValPtr, IdxList: Builder.getInt32(C: i));
4654 Address Addr =
4655 Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(Quantity: 8));
4656 ToRet = Builder.CreateStore(Val: Builder.CreateExtractValue(Agg: Val, Idxs: i), Addr);
4657 }
4658 return ToRet;
4659 }
4660
4661 // Load 8 i64 words from ValPtr, and store them to the address
4662 // via an LLVM intrinsic.
4663 SmallVector<llvm::Value *, 9> Args;
4664 Args.push_back(Elt: MemAddr);
4665 for (size_t i = 0; i < 8; i++) {
4666 llvm::Value *ValOffsetPtr =
4667 Builder.CreateGEP(Ty: Int64Ty, Ptr: ValPtr, IdxList: Builder.getInt32(C: i));
4668 Address Addr = Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(Quantity: 8));
4669 Args.push_back(Elt: Builder.CreateLoad(Addr));
4670 }
4671
4672 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
4673 ? Intrinsic::aarch64_st64b
4674 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
4675 ? Intrinsic::aarch64_st64bv
4676 : Intrinsic::aarch64_st64bv0);
4677 Function *F = CGM.getIntrinsic(IID: Intr);
4678 return Builder.CreateCall(Callee: F, Args);
4679 }
4680
4681 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
4682 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
4683
4684 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
4685 ? Intrinsic::aarch64_rndr
4686 : Intrinsic::aarch64_rndrrs);
4687 Function *F = CGM.getIntrinsic(IID: Intr);
4688 llvm::Value *Val = Builder.CreateCall(Callee: F);
4689 Value *RandomValue = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
4690 Value *Status = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
4691
4692 Address MemAddress = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4693 Builder.CreateStore(Val: RandomValue, Addr: MemAddress);
4694 Status = Builder.CreateZExt(V: Status, DestTy: Int32Ty);
4695 return Status;
4696 }
4697
4698 if (BuiltinID == clang::AArch64::BI__clear_cache) {
4699 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
4700 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
4701 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache, Tys: {CGM.DefaultPtrTy});
4702 return Builder.CreateCall(Callee: F, Args: {Begin, End});
4703 }
4704
4705 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4706 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
4707 getContext().getTypeSize(T: E->getType()) == 128) {
4708 Function *F =
4709 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4710 ? Intrinsic::aarch64_ldaxp
4711 : Intrinsic::aarch64_ldxp);
4712
4713 Value *LdPtr = EmitScalarExpr(E: E->getArg(Arg: 0));
4714 Value *Val = Builder.CreateCall(Callee: F, Args: LdPtr, Name: "ldxp");
4715
4716 Value *Val0 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
4717 Value *Val1 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
4718 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 128);
4719 Val0 = Builder.CreateZExt(V: Val0, DestTy: Int128Ty);
4720 Val1 = Builder.CreateZExt(V: Val1, DestTy: Int128Ty);
4721
4722 Value *ShiftCst = llvm::ConstantInt::get(Ty: Int128Ty, V: 64);
4723 Val = Builder.CreateShl(LHS: Val0, RHS: ShiftCst, Name: "shl", HasNUW: true /* nuw */);
4724 Val = Builder.CreateOr(LHS: Val, RHS: Val1);
4725 return Builder.CreateBitCast(V: Val, DestTy: ConvertType(T: E->getType()));
4726 } else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4727 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
4728 Value *LoadAddr = EmitScalarExpr(E: E->getArg(Arg: 0));
4729
4730 QualType Ty = E->getType();
4731 llvm::Type *RealResTy = ConvertType(T: Ty);
4732 llvm::Type *IntTy =
4733 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
4734
4735 Function *F =
4736 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4737 ? Intrinsic::aarch64_ldaxr
4738 : Intrinsic::aarch64_ldxr,
4739 Tys: DefaultPtrTy);
4740 CallInst *Val = Builder.CreateCall(Callee: F, Args: LoadAddr, Name: "ldxr");
4741 Val->addParamAttr(
4742 ArgNo: 0, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: IntTy));
4743
4744 if (RealResTy->isPointerTy())
4745 return Builder.CreateIntToPtr(V: Val, DestTy: RealResTy);
4746
4747 llvm::Type *IntResTy = llvm::IntegerType::get(
4748 C&: getLLVMContext(), NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: RealResTy));
4749 return Builder.CreateBitCast(V: Builder.CreateTruncOrBitCast(V: Val, DestTy: IntResTy),
4750 DestTy: RealResTy);
4751 }
4752
4753 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4754 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
4755 getContext().getTypeSize(T: E->getArg(Arg: 0)->getType()) == 128) {
4756 Function *F =
4757 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4758 ? Intrinsic::aarch64_stlxp
4759 : Intrinsic::aarch64_stxp);
4760 llvm::Type *STy = llvm::StructType::get(elt1: Int64Ty, elts: Int64Ty);
4761
4762 Address Tmp = CreateMemTempWithoutCast(T: E->getArg(Arg: 0)->getType());
4763 EmitAnyExprToMem(E: E->getArg(Arg: 0), Location: Tmp, Quals: Qualifiers(), /*init*/ IsInitializer: true);
4764
4765 Tmp = Tmp.withElementType(ElemTy: STy);
4766 llvm::Value *Val = Builder.CreateLoad(Addr: Tmp);
4767
4768 Value *Arg0 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
4769 Value *Arg1 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
4770 Value *StPtr = EmitScalarExpr(E: E->getArg(Arg: 1));
4771 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1, StPtr}, Name: "stxp");
4772 }
4773
4774 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4775 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
4776 Value *StoreVal = EmitScalarExpr(E: E->getArg(Arg: 0));
4777 Value *StoreAddr = EmitScalarExpr(E: E->getArg(Arg: 1));
4778
4779 QualType Ty = E->getArg(Arg: 0)->getType();
4780 llvm::Type *StoreTy =
4781 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
4782
4783 if (StoreVal->getType()->isPointerTy())
4784 StoreVal = Builder.CreatePtrToInt(V: StoreVal, DestTy: Int64Ty);
4785 else {
4786 llvm::Type *IntTy = llvm::IntegerType::get(
4787 C&: getLLVMContext(),
4788 NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: StoreVal->getType()));
4789 StoreVal = Builder.CreateBitCast(V: StoreVal, DestTy: IntTy);
4790 StoreVal = Builder.CreateZExtOrBitCast(V: StoreVal, DestTy: Int64Ty);
4791 }
4792
4793 Function *F =
4794 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4795 ? Intrinsic::aarch64_stlxr
4796 : Intrinsic::aarch64_stxr,
4797 Tys: StoreAddr->getType());
4798 CallInst *CI = Builder.CreateCall(Callee: F, Args: {StoreVal, StoreAddr}, Name: "stxr");
4799 CI->addParamAttr(
4800 ArgNo: 1, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: StoreTy));
4801 return CI;
4802 }
4803
4804 if (BuiltinID == clang::AArch64::BI__getReg ||
4805 BuiltinID == clang::AArch64::BI__setReg) {
4806 Expr::EvalResult Result;
4807 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4808 llvm_unreachable("Sema will ensure that the parameter is constant");
4809
4810 llvm::APSInt Value = Result.Val.getInt();
4811 LLVMContext &Context = CGM.getLLVMContext();
4812 std::string Reg = Value == 31 ? "sp" : "x" + toString(I: Value, Radix: 10);
4813
4814 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Str: Reg)};
4815 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
4816 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
4817
4818 CallInst *CI;
4819 if (BuiltinID == clang::AArch64::BI__getReg) {
4820 llvm::Function *F =
4821 CGM.getIntrinsic(IID: Intrinsic::read_volatile_register, Tys: {Int64Ty});
4822 CI = Builder.CreateCall(Callee: F, Args: Metadata);
4823 } else {
4824 llvm::Function *F =
4825 CGM.getIntrinsic(IID: Intrinsic::write_volatile_register, Tys: {Int64Ty});
4826 CI = Builder.CreateCall(Callee: F, Args: {Metadata, EmitScalarExpr(E: E->getArg(Arg: 1))});
4827 }
4828 return CI;
4829 }
4830
4831 if (BuiltinID == clang::AArch64::BI__getRegFp ||
4832 BuiltinID == clang::AArch64::BI__setRegFp) {
4833 Expr::EvalResult Result;
4834 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4835 llvm_unreachable("Sema will ensure that the parameter is constant");
4836
4837 llvm::APSInt Value = Result.Val.getInt();
4838 LLVMContext &Context = CGM.getLLVMContext();
4839 std::string Reg = "d" + toString(I: Value, Radix: 10);
4840
4841 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Str: Reg)};
4842 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
4843 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
4844
4845 llvm::Value *Ret;
4846 if (BuiltinID == clang::AArch64::BI__getRegFp) {
4847 llvm::Function *F =
4848 CGM.getIntrinsic(IID: Intrinsic::read_volatile_register, Tys: {Int64Ty});
4849 llvm::Value *Bits = Builder.CreateCall(Callee: F, Args: Metadata);
4850 Ret = Builder.CreateBitCast(V: Bits, DestTy: llvm::Type::getDoubleTy(C&: Context));
4851 } else {
4852 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
4853 llvm::Value *Bits = Builder.CreateBitCast(V: Val, DestTy: Int64Ty);
4854 llvm::Function *F =
4855 CGM.getIntrinsic(IID: Intrinsic::write_volatile_register, Tys: {Int64Ty});
4856 Ret = Builder.CreateCall(Callee: F, Args: {Metadata, Bits});
4857 }
4858 return Ret;
4859 }
4860
4861 if (BuiltinID == clang::AArch64::BI__break) {
4862 Expr::EvalResult Result;
4863 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4864 llvm_unreachable("Sema will ensure that the parameter is constant");
4865
4866 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_break);
4867 return Builder.CreateCall(Callee: F, Args: {EmitScalarExpr(E: E->getArg(Arg: 0))});
4868 }
4869
4870 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
4871 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_clrex);
4872 return Builder.CreateCall(Callee: F);
4873 }
4874
4875 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
4876 return Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
4877 SSID: llvm::SyncScope::SingleThread);
4878
4879 // CRC32
4880 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
4881 switch (BuiltinID) {
4882 case clang::AArch64::BI__builtin_arm_crc32b:
4883 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
4884 case clang::AArch64::BI__builtin_arm_crc32cb:
4885 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
4886 case clang::AArch64::BI__builtin_arm_crc32h:
4887 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
4888 case clang::AArch64::BI__builtin_arm_crc32ch:
4889 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
4890 case clang::AArch64::BI__builtin_arm_crc32w:
4891 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
4892 case clang::AArch64::BI__builtin_arm_crc32cw:
4893 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
4894 case clang::AArch64::BI__builtin_arm_crc32d:
4895 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
4896 case clang::AArch64::BI__builtin_arm_crc32cd:
4897 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
4898 }
4899
4900 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4901 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4902 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4903 Function *F = CGM.getIntrinsic(IID: CRCIntrinsicID);
4904
4905 llvm::Type *DataTy = F->getFunctionType()->getParamType(i: 1);
4906 Arg1 = Builder.CreateZExtOrBitCast(V: Arg1, DestTy: DataTy);
4907
4908 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1});
4909 }
4910
4911 // Memory Operations (MOPS)
4912 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
4913 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
4914 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
4915 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
4916 Val = Builder.CreateTrunc(V: Val, DestTy: Int8Ty);
4917 Size = Builder.CreateIntCast(V: Size, DestTy: Int64Ty, isSigned: false);
4918 return Builder.CreateCall(
4919 Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_mops_memset_tag), Args: {Dst, Val, Size});
4920 }
4921
4922 if (BuiltinID == AArch64::BI__builtin_arm_range_prefetch ||
4923 BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
4924 return EmitRangePrefetchBuiltin(CGF&: *this, BuiltinID, E);
4925
4926 switch (BuiltinID) {
4927 default:
4928 break;
4929 case AArch64::BI__builtin_arm_atomic_store_with_hint:
4930 return EmitAtomicStoreWithHintBuiltin(CGF&: *this, BuiltinID, E);
4931 case AArch64::BI__builtin_arm_atomic_fetch_add_with_hint:
4932 return EmitAtomicFetchWithHintBuiltin(CGF&: *this, E, Op: AtomicRMWInst::Add);
4933 case AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint:
4934 return EmitAtomicFetchWithHintBuiltin(CGF&: *this, E, Op: AtomicRMWInst::Sub);
4935 case AArch64::BI__builtin_arm_atomic_fetch_and_with_hint:
4936 return EmitAtomicFetchWithHintBuiltin(CGF&: *this, E, Op: AtomicRMWInst::And);
4937 case AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint:
4938 return EmitAtomicFetchWithHintBuiltin(CGF&: *this, E, Op: AtomicRMWInst::Xor);
4939 case AArch64::BI__builtin_arm_atomic_fetch_or_with_hint:
4940 return EmitAtomicFetchWithHintBuiltin(CGF&: *this, E, Op: AtomicRMWInst::Or);
4941 }
4942
4943 // Memory Tagging Extensions (MTE) Intrinsics
4944 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
4945 switch (BuiltinID) {
4946 case clang::AArch64::BI__builtin_arm_irg:
4947 MTEIntrinsicID = Intrinsic::aarch64_irg; break;
4948 case clang::AArch64::BI__builtin_arm_addg:
4949 MTEIntrinsicID = Intrinsic::aarch64_addg; break;
4950 case clang::AArch64::BI__builtin_arm_gmi:
4951 MTEIntrinsicID = Intrinsic::aarch64_gmi; break;
4952 case clang::AArch64::BI__builtin_arm_ldg:
4953 MTEIntrinsicID = Intrinsic::aarch64_ldg; break;
4954 case clang::AArch64::BI__builtin_arm_stg:
4955 MTEIntrinsicID = Intrinsic::aarch64_stg; break;
4956 case clang::AArch64::BI__builtin_arm_subp:
4957 MTEIntrinsicID = Intrinsic::aarch64_subp; break;
4958 }
4959
4960 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_lscale) {
4961 Value *FPM = EmitScalarExpr(E: E->getArg(Arg: 0));
4962 Value *Scale = EmitScalarExpr(E: E->getArg(Arg: 1));
4963 Scale = Builder.CreateAnd(LHS: Scale, RHS: Builder.getInt64(C: 0x7f));
4964
4965 Value *MaskedFPM = Builder.CreateAnd(LHS: FPM, RHS: Builder.getInt64(C: ~0x7f0000ULL));
4966 Value *ShiftedScale = Builder.CreateShl(LHS: Scale, RHS: Builder.getInt64(C: 16));
4967
4968 return Builder.CreateOr(LHS: MaskedFPM, RHS: ShiftedScale);
4969 }
4970 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_nscale) {
4971 Value *FPM = EmitScalarExpr(E: E->getArg(Arg: 0));
4972 Value *Scale = EmitScalarExpr(E: E->getArg(Arg: 1));
4973 Scale = Builder.CreateAnd(LHS: Scale, RHS: Builder.getInt64(C: 0xff));
4974
4975 Value *MaskedFPM = Builder.CreateAnd(LHS: FPM, RHS: Builder.getInt64(C: ~0xff000000ULL));
4976 Value *ShiftedScale = Builder.CreateShl(LHS: Scale, RHS: Builder.getInt64(C: 24));
4977
4978 return Builder.CreateOr(LHS: MaskedFPM, RHS: ShiftedScale);
4979 }
4980 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_lscale2) {
4981 Value *FPM = EmitScalarExpr(E: E->getArg(Arg: 0));
4982 Value *Scale = EmitScalarExpr(E: E->getArg(Arg: 1));
4983 Scale = Builder.CreateAnd(LHS: Scale, RHS: Builder.getInt64(C: 0x3f));
4984
4985 Value *LowFPM = Builder.CreateAnd(LHS: FPM, RHS: Builder.getInt64(C: ~0x3f00000000ULL));
4986 Value *ShiftedScale = Builder.CreateShl(LHS: Scale, RHS: Builder.getInt64(C: 32));
4987
4988 return Builder.CreateOr(LHS: LowFPM, RHS: ShiftedScale);
4989 }
4990
4991 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
4992 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
4993 Value *Pointer = EmitScalarExpr(E: E->getArg(Arg: 0));
4994 Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 1));
4995 assert(Mask->getType()->getScalarSizeInBits() == 64 &&
4996 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
4997 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
4998 Args: {Pointer, Mask});
4999 }
5000 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
5001 Value *Pointer = EmitScalarExpr(E: E->getArg(Arg: 0));
5002 Value *TagOffset = EmitScalarExpr(E: E->getArg(Arg: 1));
5003
5004 TagOffset = Builder.CreateZExt(V: TagOffset, DestTy: Int64Ty);
5005 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
5006 Args: {Pointer, TagOffset});
5007 }
5008 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
5009 Value *Pointer = EmitScalarExpr(E: E->getArg(Arg: 0));
5010 Value *ExcludedMask = EmitScalarExpr(E: E->getArg(Arg: 1));
5011 assert(ExcludedMask->getType()->getScalarSizeInBits() == 64 &&
5012 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
5013 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
5014 Args: {Pointer, ExcludedMask});
5015 }
5016 // Although it is possible to supply a different return
5017 // address (first arg) to this intrinsic, for now we set
5018 // return address same as input address.
5019 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
5020 Value *TagAddress = EmitScalarExpr(E: E->getArg(Arg: 0));
5021 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
5022 Args: {TagAddress, TagAddress});
5023 }
5024 // Although it is possible to supply a different tag (to set)
5025 // to this intrinsic (as first arg), for now we supply
5026 // the tag that is in input address arg (common use case).
5027 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
5028 Value *TagAddress = EmitScalarExpr(E: E->getArg(Arg: 0));
5029 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
5030 Args: {TagAddress, TagAddress});
5031 }
5032 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
5033 Value *PointerA = EmitScalarExpr(E: E->getArg(Arg: 0));
5034 Value *PointerB = EmitScalarExpr(E: E->getArg(Arg: 1));
5035 return Builder.CreateCall(
5036 Callee: CGM.getIntrinsic(IID: MTEIntrinsicID), Args: {PointerA, PointerB});
5037 }
5038 }
5039
5040 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5041 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
5042 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5043 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
5044 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
5045 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
5046 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
5047 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
5048
5049 SpecialRegisterAccessKind AccessKind = Write;
5050 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5051 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
5052 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5053 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
5054 AccessKind = VolatileRead;
5055
5056 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
5057 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
5058
5059 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5060 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
5061
5062 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5063 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
5064
5065 llvm::Type *ValueType;
5066 llvm::Type *RegisterType = Int64Ty;
5067 if (Is32Bit) {
5068 ValueType = Int32Ty;
5069 } else if (Is128Bit) {
5070 llvm::Type *Int128Ty =
5071 llvm::IntegerType::getInt128Ty(C&: CGM.getLLVMContext());
5072 ValueType = Int128Ty;
5073 RegisterType = Int128Ty;
5074 } else if (IsPointerBuiltin) {
5075 ValueType = VoidPtrTy;
5076 } else {
5077 ValueType = Int64Ty;
5078 };
5079
5080 return EmitSpecialRegisterBuiltin(CGF&: *this, E, RegisterType, ValueType,
5081 AccessKind);
5082 }
5083
5084 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
5085 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
5086 LLVMContext &Context = CGM.getLLVMContext();
5087
5088 unsigned SysReg =
5089 E->getArg(Arg: 0)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
5090
5091 std::string SysRegStr;
5092 llvm::raw_string_ostream(SysRegStr)
5093 << (0b10 | SysReg >> 14) << ":" << ((SysReg >> 11) & 7) << ":"
5094 << ((SysReg >> 7) & 15) << ":" << ((SysReg >> 3) & 15) << ":"
5095 << (SysReg & 7);
5096
5097 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, Str: SysRegStr) };
5098 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
5099 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
5100
5101 llvm::Type *RegisterType = Int64Ty;
5102 llvm::Type *Types[] = { RegisterType };
5103
5104 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
5105 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::read_register, Tys: Types);
5106
5107 return Builder.CreateCall(Callee: F, Args: Metadata);
5108 }
5109
5110 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::write_register, Tys: Types);
5111 llvm::Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 1));
5112 llvm::Value *Result = Builder.CreateCall(Callee: F, Args: {Metadata, ArgValue});
5113
5114 return Result;
5115 }
5116
5117 if (BuiltinID == clang::AArch64::BI__sys) {
5118 unsigned SysReg =
5119 E->getArg(Arg: 0)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
5120 const unsigned Op1 = SysReg >> 11;
5121 const unsigned CRn = (SysReg >> 7) & 0xf;
5122 const unsigned CRm = (SysReg >> 3) & 0xf;
5123 const unsigned Op2 = SysReg & 0x7;
5124
5125 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_sys),
5126 Args: {Builder.getInt32(C: Op1), Builder.getInt32(C: CRn),
5127 Builder.getInt32(C: CRm), Builder.getInt32(C: Op2),
5128 EmitScalarExpr(E: E->getArg(Arg: 1))});
5129
5130 // Return 0 for convenience, even though MSVC returns some other undefined
5131 // value.
5132 return ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
5133 }
5134
5135 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
5136 llvm::Function *F =
5137 CGM.getIntrinsic(IID: Intrinsic::addressofreturnaddress, Tys: AllocaInt8PtrTy);
5138 return Builder.CreateCall(Callee: F);
5139 }
5140
5141 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
5142 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: AllocaInt8PtrTy);
5143 return Builder.CreateCall(Callee: F);
5144 }
5145
5146 if (BuiltinID == clang::AArch64::BI__mulh ||
5147 BuiltinID == clang::AArch64::BI__umulh) {
5148 llvm::Type *ResType = ConvertType(T: E->getType());
5149 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 128);
5150
5151 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
5152 Value *LHS =
5153 Builder.CreateIntCast(V: EmitScalarExpr(E: E->getArg(Arg: 0)), DestTy: Int128Ty, isSigned: IsSigned);
5154 Value *RHS =
5155 Builder.CreateIntCast(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: Int128Ty, isSigned: IsSigned);
5156
5157 Value *MulResult, *HigherBits;
5158 if (IsSigned) {
5159 MulResult = Builder.CreateNSWMul(LHS, RHS);
5160 HigherBits = Builder.CreateAShr(LHS: MulResult, RHS: 64);
5161 } else {
5162 MulResult = Builder.CreateNUWMul(LHS, RHS);
5163 HigherBits = Builder.CreateLShr(LHS: MulResult, RHS: 64);
5164 }
5165 HigherBits = Builder.CreateIntCast(V: HigherBits, DestTy: ResType, isSigned: IsSigned);
5166
5167 return HigherBits;
5168 }
5169
5170 if (BuiltinID == AArch64::BI__writex18byte ||
5171 BuiltinID == AArch64::BI__writex18word ||
5172 BuiltinID == AArch64::BI__writex18dword ||
5173 BuiltinID == AArch64::BI__writex18qword) {
5174 // Process the args first
5175 Value *OffsetArg = EmitScalarExpr(E: E->getArg(Arg: 0));
5176 Value *DataArg = EmitScalarExpr(E: E->getArg(Arg: 1));
5177
5178 // Read x18 as i8*
5179 llvm::Value *X18 = readX18AsPtr(CGF&: *this);
5180
5181 // Store val at x18 + offset
5182 Value *Offset = Builder.CreateZExt(V: OffsetArg, DestTy: Int64Ty);
5183 Value *Ptr = Builder.CreateGEP(Ty: Int8Ty, Ptr: X18, IdxList: Offset);
5184 StoreInst *Store =
5185 Builder.CreateAlignedStore(Val: DataArg, Addr: Ptr, Align: CharUnits::One());
5186 return Store;
5187 }
5188
5189 if (BuiltinID == AArch64::BI__readx18byte ||
5190 BuiltinID == AArch64::BI__readx18word ||
5191 BuiltinID == AArch64::BI__readx18dword ||
5192 BuiltinID == AArch64::BI__readx18qword) {
5193 // Process the args first
5194 Value *OffsetArg = EmitScalarExpr(E: E->getArg(Arg: 0));
5195
5196 // Read x18 as i8*
5197 llvm::Value *X18 = readX18AsPtr(CGF&: *this);
5198
5199 // Load x18 + offset
5200 Value *Offset = Builder.CreateZExt(V: OffsetArg, DestTy: Int64Ty);
5201 Value *Ptr = Builder.CreateGEP(Ty: Int8Ty, Ptr: X18, IdxList: Offset);
5202 llvm::Type *IntTy = ConvertType(T: E->getType());
5203 LoadInst *Load = Builder.CreateAlignedLoad(Ty: IntTy, Addr: Ptr, Align: CharUnits::One());
5204 return Load;
5205 }
5206
5207 if (BuiltinID == AArch64::BI__addx18byte ||
5208 BuiltinID == AArch64::BI__addx18word ||
5209 BuiltinID == AArch64::BI__addx18dword ||
5210 BuiltinID == AArch64::BI__addx18qword ||
5211 BuiltinID == AArch64::BI__incx18byte ||
5212 BuiltinID == AArch64::BI__incx18word ||
5213 BuiltinID == AArch64::BI__incx18dword ||
5214 BuiltinID == AArch64::BI__incx18qword) {
5215 llvm::Type *IntTy;
5216 bool isIncrement;
5217 switch (BuiltinID) {
5218 case AArch64::BI__incx18byte:
5219 IntTy = Int8Ty;
5220 isIncrement = true;
5221 break;
5222 case AArch64::BI__incx18word:
5223 IntTy = Int16Ty;
5224 isIncrement = true;
5225 break;
5226 case AArch64::BI__incx18dword:
5227 IntTy = Int32Ty;
5228 isIncrement = true;
5229 break;
5230 case AArch64::BI__incx18qword:
5231 IntTy = Int64Ty;
5232 isIncrement = true;
5233 break;
5234 default:
5235 IntTy = ConvertType(T: E->getArg(Arg: 1)->getType());
5236 isIncrement = false;
5237 break;
5238 }
5239 // Process the args first
5240 Value *OffsetArg = EmitScalarExpr(E: E->getArg(Arg: 0));
5241 Value *ValToAdd =
5242 isIncrement ? ConstantInt::get(Ty: IntTy, V: 1) : EmitScalarExpr(E: E->getArg(Arg: 1));
5243
5244 // Read x18 as i8*
5245 llvm::Value *X18 = readX18AsPtr(CGF&: *this);
5246
5247 // Load x18 + offset
5248 Value *Offset = Builder.CreateZExt(V: OffsetArg, DestTy: Int64Ty);
5249 Value *Ptr = Builder.CreateGEP(Ty: Int8Ty, Ptr: X18, IdxList: Offset);
5250 LoadInst *Load = Builder.CreateAlignedLoad(Ty: IntTy, Addr: Ptr, Align: CharUnits::One());
5251
5252 // Add values
5253 Value *AddResult = Builder.CreateAdd(LHS: Load, RHS: ValToAdd);
5254
5255 // Store val at x18 + offset
5256 StoreInst *Store =
5257 Builder.CreateAlignedStore(Val: AddResult, Addr: Ptr, Align: CharUnits::One());
5258 return Store;
5259 }
5260
5261 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
5262 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
5263 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
5264 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
5265 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
5266 llvm::Type *RetTy = ConvertType(T: E->getType());
5267 return Builder.CreateBitCast(V: Arg, DestTy: RetTy);
5268 }
5269
5270 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5271 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5272 BuiltinID == AArch64::BI_CountLeadingZeros ||
5273 BuiltinID == AArch64::BI_CountLeadingZeros64) {
5274 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
5275 llvm::Type *ArgType = Arg->getType();
5276
5277 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5278 BuiltinID == AArch64::BI_CountLeadingOnes64)
5279 Arg = Builder.CreateXor(LHS: Arg, RHS: Constant::getAllOnesValue(Ty: ArgType));
5280
5281 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
5282 Value *Result = Builder.CreateCall(Callee: F, Args: {Arg, Builder.getInt1(V: false)});
5283
5284 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5285 BuiltinID == AArch64::BI_CountLeadingZeros64)
5286 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5287 return Result;
5288 }
5289
5290 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
5291 BuiltinID == AArch64::BI_CountLeadingSigns64) {
5292 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
5293
5294 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
5295 ? CGM.getIntrinsic(IID: Intrinsic::aarch64_cls)
5296 : CGM.getIntrinsic(IID: Intrinsic::aarch64_cls64);
5297
5298 Value *Result = Builder.CreateCall(Callee: F, Args: Arg, Name: "cls");
5299 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
5300 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5301 return Result;
5302 }
5303
5304 if (BuiltinID == AArch64::BI_CountOneBits ||
5305 BuiltinID == AArch64::BI_CountOneBits64) {
5306 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
5307 llvm::Type *ArgType = ArgValue->getType();
5308 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
5309
5310 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
5311 if (BuiltinID == AArch64::BI_CountOneBits64)
5312 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5313 return Result;
5314 }
5315
5316 if (BuiltinID == AArch64::BI_CountTrailingZeros ||
5317 BuiltinID == AArch64::BI_CountTrailingZeros64) {
5318 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
5319 llvm::Type *ArgType = ArgValue->getType();
5320 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
5321
5322 // MSVC leaves 0 undefined; use false for predictable codegen
5323 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getInt1(V: false)});
5324 if (BuiltinID == AArch64::BI_CountTrailingZeros64)
5325 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5326 return Result;
5327 }
5328
5329 if (BuiltinID == AArch64::BI__prefetch) {
5330 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5331 Value *RW = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
5332 Value *Locality = ConstantInt::get(Ty: Int32Ty, V: 3);
5333 Value *Data = llvm::ConstantInt::get(Ty: Int32Ty, V: 1);
5334 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
5335 return Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, Data});
5336 }
5337
5338 if (BuiltinID == AArch64::BI__prefetch2) {
5339 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5340 llvm::APSInt PrfOp = E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: CGM.getContext());
5341 // Decode 5-bit PRFM encoding: bits[4:3]=type, bits[2:1]=target,
5342 // bit[0]=policy
5343 // type: PLD=0(load), PLI=1(instr), PST=2(store)
5344 // target: L1=0, L2=1, L3=2
5345 // policy: KEEP=0, STRM=1
5346 uint64_t Op = PrfOp.getZExtValue();
5347 uint64_t Type = (Op >> 3) & 0x3;
5348 uint64_t Target = (Op >> 1) & 0x3;
5349 uint64_t Policy = Op & 0x1;
5350 Value *RW = Builder.getInt32(C: Type == 2 ? 1 : 0);
5351 Value *Local = Builder.getInt32(C: Target);
5352 Value *IsStream = Builder.getInt32(C: Policy);
5353 Value *IsData = Builder.getInt32(C: Type == 1 ? 0 : 1);
5354 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_prefetch);
5355 return Builder.CreateCall(Callee: F, Args: {Address, RW, Local, IsStream, IsData});
5356 }
5357
5358 if (BuiltinID == AArch64::BI__hlt) {
5359 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_hlt);
5360 Builder.CreateCall(Callee: F, Args: {EmitScalarExpr(E: E->getArg(Arg: 0))});
5361
5362 // FIXME: MSVC documents __hlt as taking further arguments in X0-X3 and
5363 // returning the value in X0, like __hvc/__svc below. This ignores the
5364 // extra arguments and returns 0.
5365 return ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
5366 }
5367
5368 if (BuiltinID == AArch64::BI__hvc || BuiltinID == AArch64::BI__svc) {
5369 unsigned IID = BuiltinID == AArch64::BI__svc ? Intrinsic::aarch64_svc
5370 : Intrinsic::aarch64_hvc;
5371 // The first argument is the instruction immediate; it must be a constant
5372 // (ImmArg on the intrinsic, encoded in the instruction). The remaining
5373 // arguments (at most four, enforced by Sema) are passed in X0-X3, widened
5374 // to 64 bits. The intrinsic takes exactly four register operands, so any
5375 // unused trailing ones are passed as poison and dropped during lowering.
5376 SmallVector<Value *, 5> Args{Builder.getInt32(
5377 C: GetIntegerConstantValue<uint32_t>(E: E->getArg(Arg: 0), Context&: getContext()))};
5378 for (unsigned I = 1, N = E->getNumArgs(); I < N; ++I) {
5379 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: I));
5380 llvm::Type *ArgTy = Arg->getType();
5381 if (ArgTy->isPointerTy())
5382 Arg = Builder.CreatePtrToInt(V: Arg, DestTy: Int64Ty);
5383 else if (ArgTy->isFloatingPointTy())
5384 // Reinterpret the bits into the integer register, matching MSVC (e.g.
5385 // "fmov x0, d0" for a double).
5386 Arg = Builder.CreateZExtOrTrunc(
5387 V: Builder.CreateBitCast(
5388 V: Arg, DestTy: Builder.getIntNTy(N: ArgTy->getPrimitiveSizeInBits())),
5389 DestTy: Int64Ty);
5390 else
5391 Arg = Builder.CreateIntCast(
5392 V: Arg, DestTy: Int64Ty, isSigned: E->getArg(Arg: I)->getType()->isSignedIntegerType());
5393 Args.push_back(Elt: Arg);
5394 }
5395 while (Args.size() < 5)
5396 Args.push_back(Elt: llvm::PoisonValue::get(T: Int64Ty));
5397 Value *Call = Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args);
5398 // MSVC returns unsigned int, i.e. the low 32 bits of the X0 result.
5399 return Builder.CreateTrunc(V: Call, DestTy: Int32Ty);
5400 }
5401
5402 if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
5403 return Builder.CreateFPTrunc(
5404 V: Builder.CreateBitCast(V: EmitScalarExpr(E: E->getArg(Arg: 0)),
5405 DestTy: Builder.getFloatTy()),
5406 DestTy: Builder.getBFloatTy());
5407
5408 // Handle MSVC intrinsics before argument evaluation to prevent double
5409 // evaluation.
5410 if (std::optional<MSVCIntrin> MsvcIntId =
5411 translateAarch64ToMsvcIntrin(BuiltinID))
5412 return EmitMSVCBuiltinExpr(BuiltinID: *MsvcIntId, E);
5413
5414 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
5415 auto It = llvm::find_if(Range: NEONEquivalentIntrinsicMap, P: [BuiltinID](auto &P) {
5416 return P.first == BuiltinID;
5417 });
5418 if (It != end(arr: NEONEquivalentIntrinsicMap))
5419 BuiltinID = It->second;
5420
5421 // Check whether this is an SISD builtin.
5422 auto SISDMap = ArrayRef(AArch64SISDIntrinsicMap);
5423 const ARMNeonVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap(
5424 IntrinsicMap: SISDMap, BuiltinID, MapProvenSorted&: AArch64SISDIntrinsicsProvenSorted);
5425 bool IsSISD = (Builtin != nullptr);
5426
5427 // Find out if any arguments are required to be integer constant
5428 // expressions.
5429 unsigned ICEArguments = 0;
5430 ASTContext::GetBuiltinTypeError Error;
5431 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
5432 assert(Error == ASTContext::GE_None && "Should not codegen an error");
5433
5434 llvm::SmallVector<Value*, 4> Ops;
5435 Address PtrOp0 = Address::invalid();
5436 // Note the assumption that SISD intrinsics do not contain extra arguments.
5437 // TODO: Fold this into a single function call instead of, effectively, two
5438 // separate checks.
5439 bool HasExtraArg = !IsSISD && CodeGenUtils::hasExtraNeonArgument(BuiltinID);
5440 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
5441 for (unsigned i = 0, e = NumArgs; i != e; i++) {
5442 if (i == 0) {
5443 switch (BuiltinID) {
5444 case NEON::BI__builtin_neon_vld1_v:
5445 case NEON::BI__builtin_neon_vld1q_v:
5446 case NEON::BI__builtin_neon_vld1_dup_v:
5447 case NEON::BI__builtin_neon_vld1q_dup_v:
5448 case NEON::BI__builtin_neon_vld1_lane_v:
5449 case NEON::BI__builtin_neon_vld1q_lane_v:
5450 case NEON::BI__builtin_neon_vst1_v:
5451 case NEON::BI__builtin_neon_vst1q_v:
5452 case NEON::BI__builtin_neon_vst1_lane_v:
5453 case NEON::BI__builtin_neon_vst1q_lane_v:
5454 case NEON::BI__builtin_neon_vldap1_lane_s64:
5455 case NEON::BI__builtin_neon_vldap1q_lane_s64:
5456 case NEON::BI__builtin_neon_vstl1_lane_s64:
5457 case NEON::BI__builtin_neon_vstl1q_lane_s64:
5458 // Get the alignment for the argument in addition to the value;
5459 // we'll use it later.
5460 PtrOp0 = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5461 Ops.push_back(Elt: PtrOp0.emitRawPointer(CGF&: *this));
5462 continue;
5463 }
5464 }
5465 Ops.push_back(Elt: EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E));
5466 }
5467
5468 if (Builtin) {
5469 Value *Result = EmitCommonNeonSISDBuiltinExpr(CGF&: *this, SISDInfo: *Builtin, Ops, E);
5470 assert(Result && "SISD intrinsic should have been handled");
5471 return Result;
5472 }
5473
5474 const Expr *Arg = E->getArg(Arg: E->getNumArgs()-1);
5475 NeonTypeFlags Type(0);
5476 if (std::optional<llvm::APSInt> Result =
5477 Arg->getIntegerConstantExpr(Ctx: getContext()))
5478 // Determine the type of this overloaded NEON intrinsic.
5479 Type = NeonTypeFlags(Result->getZExtValue());
5480
5481 bool usgn = Type.isUnsigned();
5482 bool quad = Type.isQuad();
5483 unsigned Int;
5484
5485 // Not all intrinsics handled by the common case work for AArch64 yet, so only
5486 // defer to common code if it's been added to our special map.
5487 Builtin =
5488 findARMVectorIntrinsicInMap(IntrinsicMap: ArrayRef(AArch64SIMDIntrinsicMap), BuiltinID,
5489 MapProvenSorted&: AArch64SIMDIntrinsicsProvenSorted);
5490
5491 if (Builtin)
5492 return EmitCommonNeonBuiltinExpr(
5493 BuiltinID: Builtin->BuiltinID, LLVMIntrinsic: Builtin->LLVMIntrinsic, AltLLVMIntrinsic: Builtin->AltLLVMIntrinsic,
5494 NameHint: Builtin->NameHint, Modifier: Builtin->TypeModifier, E, Ops,
5495 /*never use addresses*/ PtrOp0: Address::invalid(), PtrOp1: Address::invalid(), Arch);
5496
5497 if (Value *V = EmitAArch64TblBuiltinExpr(CGF&: *this, BuiltinID, E, Ops, Arch))
5498 return V;
5499
5500 // Handle non-overloaded intrinsics first.
5501 switch (BuiltinID) {
5502 default: break;
5503 case NEON::BI__builtin_neon_vabsh_f16:
5504 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::fabs, Tys: HalfTy), Ops, name: "vabs");
5505 case NEON::BI__builtin_neon_vaddq_p128: {
5506 llvm::Type *Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags::Poly128);
5507 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
5508 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
5509 Ops[0] = Builder.CreateXor(LHS: Ops[0], RHS: Ops[1]);
5510 llvm::Type *Int128Ty = llvm::Type::getIntNTy(C&: getLLVMContext(), N: 128);
5511 return Builder.CreateBitCast(V: Ops[0], DestTy: Int128Ty);
5512 }
5513 case NEON::BI__builtin_neon_vldrq_p128: {
5514 llvm::Type *Int128Ty = llvm::Type::getIntNTy(C&: getLLVMContext(), N: 128);
5515 return Builder.CreateAlignedLoad(Ty: Int128Ty, Addr: Ops[0],
5516 Align: CharUnits::fromQuantity(Quantity: 16));
5517 }
5518 case NEON::BI__builtin_neon_vstrq_p128: {
5519 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
5520 }
5521 case NEON::BI__builtin_neon_vcvts_f32_u32:
5522 case NEON::BI__builtin_neon_vcvtd_f64_u64:
5523 usgn = true;
5524 [[fallthrough]];
5525 case NEON::BI__builtin_neon_vcvts_f32_s32:
5526 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5527 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5528 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
5529 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
5530 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: InTy);
5531 if (usgn)
5532 return Builder.CreateUIToFP(V: Ops[0], DestTy: FTy);
5533 return Builder.CreateSIToFP(V: Ops[0], DestTy: FTy);
5534 }
5535 case NEON::BI__builtin_neon_vcvth_f16_u16:
5536 case NEON::BI__builtin_neon_vcvth_f16_u32:
5537 case NEON::BI__builtin_neon_vcvth_f16_u64:
5538 usgn = true;
5539 [[fallthrough]];
5540 case NEON::BI__builtin_neon_vcvth_f16_s16:
5541 case NEON::BI__builtin_neon_vcvth_f16_s32:
5542 case NEON::BI__builtin_neon_vcvth_f16_s64: {
5543 llvm::Type *FTy = HalfTy;
5544 llvm::Type *InTy;
5545 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
5546 InTy = Int64Ty;
5547 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
5548 InTy = Int32Ty;
5549 else
5550 InTy = Int16Ty;
5551 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: InTy);
5552 if (usgn)
5553 return Builder.CreateUIToFP(V: Ops[0], DestTy: FTy);
5554 return Builder.CreateSIToFP(V: Ops[0], DestTy: FTy);
5555 }
5556 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5557 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5558 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5559 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5560 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5561 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5562 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5563 case NEON::BI__builtin_neon_vcvtph_s16_f16: {
5564 llvm::Type *InTy = Int16Ty;
5565 llvm::Type* FTy = HalfTy;
5566 llvm::Type *Tys[2] = {InTy, FTy};
5567 switch (BuiltinID) {
5568 default: llvm_unreachable("missing builtin ID in switch!");
5569 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5570 Int = Intrinsic::aarch64_neon_fcvtau; break;
5571 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5572 Int = Intrinsic::aarch64_neon_fcvtmu; break;
5573 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5574 Int = Intrinsic::aarch64_neon_fcvtnu; break;
5575 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5576 Int = Intrinsic::aarch64_neon_fcvtpu; break;
5577 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5578 Int = Intrinsic::aarch64_neon_fcvtas; break;
5579 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5580 Int = Intrinsic::aarch64_neon_fcvtms; break;
5581 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5582 Int = Intrinsic::aarch64_neon_fcvtns; break;
5583 case NEON::BI__builtin_neon_vcvtph_s16_f16:
5584 Int = Intrinsic::aarch64_neon_fcvtps; break;
5585 }
5586 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fcvt");
5587 }
5588 case NEON::BI__builtin_neon_vcaleh_f16:
5589 case NEON::BI__builtin_neon_vcalth_f16:
5590 case NEON::BI__builtin_neon_vcageh_f16:
5591 case NEON::BI__builtin_neon_vcagth_f16: {
5592 llvm::Type* InTy = Int32Ty;
5593 llvm::Type* FTy = HalfTy;
5594 llvm::Type *Tys[2] = {InTy, FTy};
5595 switch (BuiltinID) {
5596 default: llvm_unreachable("missing builtin ID in switch!");
5597 case NEON::BI__builtin_neon_vcageh_f16:
5598 Int = Intrinsic::aarch64_neon_facge; break;
5599 case NEON::BI__builtin_neon_vcagth_f16:
5600 Int = Intrinsic::aarch64_neon_facgt; break;
5601 case NEON::BI__builtin_neon_vcaleh_f16:
5602 Int = Intrinsic::aarch64_neon_facge; std::swap(a&: Ops[0], b&: Ops[1]); break;
5603 case NEON::BI__builtin_neon_vcalth_f16:
5604 Int = Intrinsic::aarch64_neon_facgt; std::swap(a&: Ops[0], b&: Ops[1]); break;
5605 }
5606 Ops[0] = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "facg");
5607 return Builder.CreateTrunc(V: Ops[0], DestTy: Int16Ty);
5608 }
5609 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5610 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
5611 llvm::Type* InTy = Int32Ty;
5612 llvm::Type* FTy = HalfTy;
5613 llvm::Type *Tys[2] = {InTy, FTy};
5614 switch (BuiltinID) {
5615 default: llvm_unreachable("missing builtin ID in switch!");
5616 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5617 Int = Intrinsic::aarch64_neon_vcvtfp2fxs; break;
5618 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
5619 Int = Intrinsic::aarch64_neon_vcvtfp2fxu; break;
5620 }
5621 Ops[0] = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fcvth_n");
5622 return Builder.CreateTrunc(V: Ops[0], DestTy: Int16Ty);
5623 }
5624 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5625 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
5626 llvm::Type* FTy = HalfTy;
5627 llvm::Type* InTy = Int32Ty;
5628 llvm::Type *Tys[2] = {FTy, InTy};
5629 switch (BuiltinID) {
5630 default: llvm_unreachable("missing builtin ID in switch!");
5631 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5632 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
5633 Ops[0] = Builder.CreateSExt(V: Ops[0], DestTy: InTy, Name: "sext");
5634 break;
5635 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
5636 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
5637 Ops[0] = Builder.CreateZExt(V: Ops[0], DestTy: InTy);
5638 break;
5639 }
5640 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fcvth_n");
5641 }
5642 case NEON::BI__builtin_neon_vpaddd_s64: {
5643 // TODO: Isn't this handled by
5644 // EmitCommonNeonSISDBuiltinExpr?
5645 auto *Ty = llvm::FixedVectorType::get(ElementType: Int64Ty, NumElts: 2);
5646 // The vector is v2f64, so make sure it's bitcast to that.
5647 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty, Name: "v2i64");
5648 llvm::Value *Idx0 = llvm::ConstantInt::get(Ty: SizeTy, V: 0);
5649 llvm::Value *Idx1 = llvm::ConstantInt::get(Ty: SizeTy, V: 1);
5650 Value *Op0 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx0, Name: "lane0");
5651 Value *Op1 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx1, Name: "lane1");
5652 // Pairwise addition of a v2f64 into a scalar f64.
5653 return Builder.CreateAdd(LHS: Op0, RHS: Op1, Name: "vpaddd");
5654 }
5655 case NEON::BI__builtin_neon_vpaddd_f64: {
5656 auto *Ty = llvm::FixedVectorType::get(ElementType: DoubleTy, NumElts: 2);
5657 // The vector is v2f64, so make sure it's bitcast to that.
5658 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty, Name: "v2f64");
5659 llvm::Value *Idx0 = llvm::ConstantInt::get(Ty: SizeTy, V: 0);
5660 llvm::Value *Idx1 = llvm::ConstantInt::get(Ty: SizeTy, V: 1);
5661 Value *Op0 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx0, Name: "lane0");
5662 Value *Op1 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx1, Name: "lane1");
5663 // Pairwise addition of a v2f64 into a scalar f64.
5664 return Builder.CreateFAdd(L: Op0, R: Op1, Name: "vpaddd");
5665 }
5666 case NEON::BI__builtin_neon_vpadds_f32: {
5667 auto *Ty = llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 2);
5668 // The vector is v2f32, so make sure it's bitcast to that.
5669 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty, Name: "v2f32");
5670 llvm::Value *Idx0 = llvm::ConstantInt::get(Ty: SizeTy, V: 0);
5671 llvm::Value *Idx1 = llvm::ConstantInt::get(Ty: SizeTy, V: 1);
5672 Value *Op0 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx0, Name: "lane0");
5673 Value *Op1 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx1, Name: "lane1");
5674 // Pairwise addition of a v2f32 into a scalar f32.
5675 return Builder.CreateFAdd(L: Op0, R: Op1, Name: "vpaddd");
5676 }
5677 case NEON::BI__builtin_neon_vceqzd_s64:
5678 return EmitAArch64CompareBuiltinExpr(
5679 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5680 Pred: ICmpInst::ICMP_EQ, Name: "vceqz");
5681 case NEON::BI__builtin_neon_vceqzd_f64:
5682 case NEON::BI__builtin_neon_vceqzs_f32:
5683 case NEON::BI__builtin_neon_vceqzh_f16:
5684 return EmitAArch64CompareBuiltinExpr(
5685 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5686 Pred: ICmpInst::FCMP_OEQ, Name: "vceqz");
5687 case NEON::BI__builtin_neon_vcgezd_s64:
5688 return EmitAArch64CompareBuiltinExpr(
5689 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5690 Pred: ICmpInst::ICMP_SGE, Name: "vcgez");
5691 case NEON::BI__builtin_neon_vcgezd_f64:
5692 case NEON::BI__builtin_neon_vcgezs_f32:
5693 case NEON::BI__builtin_neon_vcgezh_f16:
5694 return EmitAArch64CompareBuiltinExpr(
5695 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5696 Pred: ICmpInst::FCMP_OGE, Name: "vcgez");
5697 case NEON::BI__builtin_neon_vclezd_s64:
5698 return EmitAArch64CompareBuiltinExpr(
5699 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5700 Pred: ICmpInst::ICMP_SLE, Name: "vclez");
5701 case NEON::BI__builtin_neon_vclezd_f64:
5702 case NEON::BI__builtin_neon_vclezs_f32:
5703 case NEON::BI__builtin_neon_vclezh_f16:
5704 return EmitAArch64CompareBuiltinExpr(
5705 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5706 Pred: ICmpInst::FCMP_OLE, Name: "vclez");
5707 case NEON::BI__builtin_neon_vcgtzd_s64:
5708 return EmitAArch64CompareBuiltinExpr(
5709 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5710 Pred: ICmpInst::ICMP_SGT, Name: "vcgtz");
5711 case NEON::BI__builtin_neon_vcgtzd_f64:
5712 case NEON::BI__builtin_neon_vcgtzs_f32:
5713 case NEON::BI__builtin_neon_vcgtzh_f16:
5714 return EmitAArch64CompareBuiltinExpr(
5715 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5716 Pred: ICmpInst::FCMP_OGT, Name: "vcgtz");
5717 case NEON::BI__builtin_neon_vcltzd_s64:
5718 return EmitAArch64CompareBuiltinExpr(
5719 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5720 Pred: ICmpInst::ICMP_SLT, Name: "vcltz");
5721
5722 case NEON::BI__builtin_neon_vcltzd_f64:
5723 case NEON::BI__builtin_neon_vcltzs_f32:
5724 case NEON::BI__builtin_neon_vcltzh_f16:
5725 return EmitAArch64CompareBuiltinExpr(
5726 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5727 Pred: ICmpInst::FCMP_OLT, Name: "vcltz");
5728
5729 case NEON::BI__builtin_neon_vceqzd_u64: {
5730 return EmitAArch64CompareBuiltinExpr(
5731 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5732 Pred: ICmpInst::ICMP_EQ, Name: "vceqzd");
5733 }
5734 case NEON::BI__builtin_neon_vceqd_f64:
5735 case NEON::BI__builtin_neon_vcled_f64:
5736 case NEON::BI__builtin_neon_vcltd_f64:
5737 case NEON::BI__builtin_neon_vcged_f64:
5738 case NEON::BI__builtin_neon_vcgtd_f64: {
5739 llvm::CmpInst::Predicate P;
5740 switch (BuiltinID) {
5741 default: llvm_unreachable("missing builtin ID in switch!");
5742 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
5743 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
5744 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
5745 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
5746 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
5747 }
5748 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
5749 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: DoubleTy);
5750 if (P == llvm::FCmpInst::FCMP_OEQ)
5751 Ops[0] = Builder.CreateFCmp(P, LHS: Ops[0], RHS: Ops[1]);
5752 else
5753 Ops[0] = Builder.CreateFCmpS(P, LHS: Ops[0], RHS: Ops[1]);
5754 return Builder.CreateSExt(V: Ops[0], DestTy: Int64Ty, Name: "vcmpd");
5755 }
5756 case NEON::BI__builtin_neon_vceqs_f32:
5757 case NEON::BI__builtin_neon_vcles_f32:
5758 case NEON::BI__builtin_neon_vclts_f32:
5759 case NEON::BI__builtin_neon_vcges_f32:
5760 case NEON::BI__builtin_neon_vcgts_f32: {
5761 llvm::CmpInst::Predicate P;
5762 switch (BuiltinID) {
5763 default: llvm_unreachable("missing builtin ID in switch!");
5764 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
5765 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
5766 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
5767 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
5768 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
5769 }
5770 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: FloatTy);
5771 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: FloatTy);
5772 if (P == llvm::FCmpInst::FCMP_OEQ)
5773 Ops[0] = Builder.CreateFCmp(P, LHS: Ops[0], RHS: Ops[1]);
5774 else
5775 Ops[0] = Builder.CreateFCmpS(P, LHS: Ops[0], RHS: Ops[1]);
5776 return Builder.CreateSExt(V: Ops[0], DestTy: Int32Ty, Name: "vcmpd");
5777 }
5778 case NEON::BI__builtin_neon_vceqh_f16:
5779 case NEON::BI__builtin_neon_vcleh_f16:
5780 case NEON::BI__builtin_neon_vclth_f16:
5781 case NEON::BI__builtin_neon_vcgeh_f16:
5782 case NEON::BI__builtin_neon_vcgth_f16: {
5783 llvm::CmpInst::Predicate P;
5784 switch (BuiltinID) {
5785 default: llvm_unreachable("missing builtin ID in switch!");
5786 case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ; break;
5787 case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE; break;
5788 case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT; break;
5789 case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE; break;
5790 case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT; break;
5791 }
5792 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: HalfTy);
5793 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: HalfTy);
5794 if (P == llvm::FCmpInst::FCMP_OEQ)
5795 Ops[0] = Builder.CreateFCmp(P, LHS: Ops[0], RHS: Ops[1]);
5796 else
5797 Ops[0] = Builder.CreateFCmpS(P, LHS: Ops[0], RHS: Ops[1]);
5798 return Builder.CreateSExt(V: Ops[0], DestTy: Int16Ty, Name: "vcmpd");
5799 }
5800 case NEON::BI__builtin_neon_vceqd_s64:
5801 case NEON::BI__builtin_neon_vceqd_u64:
5802 case NEON::BI__builtin_neon_vcgtd_s64:
5803 case NEON::BI__builtin_neon_vcgtd_u64:
5804 case NEON::BI__builtin_neon_vcltd_s64:
5805 case NEON::BI__builtin_neon_vcltd_u64:
5806 case NEON::BI__builtin_neon_vcged_u64:
5807 case NEON::BI__builtin_neon_vcged_s64:
5808 case NEON::BI__builtin_neon_vcled_u64:
5809 case NEON::BI__builtin_neon_vcled_s64: {
5810 llvm::CmpInst::Predicate P;
5811 switch (BuiltinID) {
5812 default: llvm_unreachable("missing builtin ID in switch!");
5813 case NEON::BI__builtin_neon_vceqd_s64:
5814 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
5815 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
5816 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
5817 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
5818 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
5819 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
5820 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
5821 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
5822 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
5823 }
5824 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Int64Ty);
5825 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty);
5826 Ops[0] = Builder.CreateICmp(P, LHS: Ops[0], RHS: Ops[1]);
5827 return Builder.CreateSExt(V: Ops[0], DestTy: Int64Ty, Name: "vceqd");
5828 }
5829 case NEON::BI__builtin_neon_vnegd_s64:
5830 return Builder.CreateNeg(V: Ops[0], Name: "vnegd");
5831 case NEON::BI__builtin_neon_vnegh_f16:
5832 return Builder.CreateFNeg(V: Ops[0], Name: "vnegh");
5833 case NEON::BI__builtin_neon_vtstd_s64:
5834 case NEON::BI__builtin_neon_vtstd_u64: {
5835 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Int64Ty);
5836 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty);
5837 Ops[0] = Builder.CreateAnd(LHS: Ops[0], RHS: Ops[1]);
5838 Ops[0] = Builder.CreateICmp(P: ICmpInst::ICMP_NE, LHS: Ops[0],
5839 RHS: llvm::Constant::getNullValue(Ty: Int64Ty));
5840 return Builder.CreateSExt(V: Ops[0], DestTy: Int64Ty, Name: "vtstd");
5841 }
5842 case NEON::BI__builtin_neon_vset_lane_i8:
5843 case NEON::BI__builtin_neon_vset_lane_i16:
5844 case NEON::BI__builtin_neon_vset_lane_i32:
5845 case NEON::BI__builtin_neon_vset_lane_i64:
5846 case NEON::BI__builtin_neon_vset_lane_bf16:
5847 case NEON::BI__builtin_neon_vset_lane_f32:
5848 case NEON::BI__builtin_neon_vsetq_lane_i8:
5849 case NEON::BI__builtin_neon_vsetq_lane_i16:
5850 case NEON::BI__builtin_neon_vsetq_lane_i32:
5851 case NEON::BI__builtin_neon_vsetq_lane_i64:
5852 case NEON::BI__builtin_neon_vsetq_lane_bf16:
5853 case NEON::BI__builtin_neon_vsetq_lane_f32:
5854 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5855 case NEON::BI__builtin_neon_vset_lane_f64:
5856 // The vector type needs a cast for the v1f64 variant.
5857 Ops[1] =
5858 Builder.CreateBitCast(V: Ops[1], DestTy: llvm::FixedVectorType::get(ElementType: DoubleTy, NumElts: 1));
5859 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5860 case NEON::BI__builtin_neon_vset_lane_mf8:
5861 case NEON::BI__builtin_neon_vsetq_lane_mf8:
5862 // The input vector type needs a cast to scalar type.
5863 Ops[0] =
5864 Builder.CreateBitCast(V: Ops[0], DestTy: llvm::Type::getInt8Ty(C&: getLLVMContext()));
5865 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5866 case NEON::BI__builtin_neon_vsetq_lane_f64:
5867 // The vector type needs a cast for the v2f64 variant.
5868 Ops[1] =
5869 Builder.CreateBitCast(V: Ops[1], DestTy: llvm::FixedVectorType::get(ElementType: DoubleTy, NumElts: 2));
5870 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5871
5872 case NEON::BI__builtin_neon_vget_lane_i8:
5873 case NEON::BI__builtin_neon_vdupb_lane_i8:
5874 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5875 case NEON::BI__builtin_neon_vgetq_lane_i8:
5876 case NEON::BI__builtin_neon_vdupb_laneq_i8:
5877 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5878 case NEON::BI__builtin_neon_vget_lane_mf8:
5879 case NEON::BI__builtin_neon_vdupb_lane_mf8:
5880 case NEON::BI__builtin_neon_vgetq_lane_mf8:
5881 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
5882 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5883 case NEON::BI__builtin_neon_vget_lane_i16:
5884 case NEON::BI__builtin_neon_vduph_lane_i16:
5885 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5886 case NEON::BI__builtin_neon_vgetq_lane_i16:
5887 case NEON::BI__builtin_neon_vduph_laneq_i16:
5888 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5889 case NEON::BI__builtin_neon_vget_lane_i32:
5890 case NEON::BI__builtin_neon_vdups_lane_i32:
5891 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5892 case NEON::BI__builtin_neon_vdups_lane_f32:
5893 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vdups_lane");
5894 case NEON::BI__builtin_neon_vgetq_lane_i32:
5895 case NEON::BI__builtin_neon_vdups_laneq_i32:
5896 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5897 case NEON::BI__builtin_neon_vget_lane_i64:
5898 case NEON::BI__builtin_neon_vdupd_lane_i64:
5899 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5900 case NEON::BI__builtin_neon_vdupd_lane_f64:
5901 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vdupd_lane");
5902 case NEON::BI__builtin_neon_vgetq_lane_i64:
5903 case NEON::BI__builtin_neon_vdupd_laneq_i64:
5904 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5905 case NEON::BI__builtin_neon_vget_lane_f32:
5906 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5907 case NEON::BI__builtin_neon_vget_lane_f64:
5908 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5909 case NEON::BI__builtin_neon_vgetq_lane_f32:
5910 case NEON::BI__builtin_neon_vdups_laneq_f32:
5911 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5912 case NEON::BI__builtin_neon_vgetq_lane_f64:
5913 case NEON::BI__builtin_neon_vdupd_laneq_f64:
5914 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5915 case NEON::BI__builtin_neon_vaddh_f16:
5916 return Builder.CreateFAdd(L: Ops[0], R: Ops[1], Name: "vaddh");
5917 case NEON::BI__builtin_neon_vsubh_f16:
5918 return Builder.CreateFSub(L: Ops[0], R: Ops[1], Name: "vsubh");
5919 case NEON::BI__builtin_neon_vmulh_f16:
5920 return Builder.CreateFMul(L: Ops[0], R: Ops[1], Name: "vmulh");
5921 case NEON::BI__builtin_neon_vdivh_f16:
5922 return Builder.CreateFDiv(L: Ops[0], R: Ops[1], Name: "vdivh");
5923 case NEON::BI__builtin_neon_vfmah_f16:
5924 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5925 return emitCallMaybeConstrainedFPBuiltin(
5926 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty: HalfTy,
5927 Args: {Ops[1], Ops[2], Ops[0]});
5928 case NEON::BI__builtin_neon_vfmsh_f16: {
5929 Value *Neg = Builder.CreateFNeg(V: Ops[1], Name: "vsubh");
5930
5931 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5932 return emitCallMaybeConstrainedFPBuiltin(
5933 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty: HalfTy,
5934 Args: {Neg, Ops[2], Ops[0]});
5935 }
5936 case NEON::BI__builtin_neon_vaddd_s64:
5937 case NEON::BI__builtin_neon_vaddd_u64:
5938 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1], Name: "vaddd");
5939 case NEON::BI__builtin_neon_vsubd_s64:
5940 case NEON::BI__builtin_neon_vsubd_u64:
5941 return Builder.CreateSub(LHS: Ops[0], RHS: Ops[1], Name: "vsubd");
5942 case NEON::BI__builtin_neon_vqdmlalh_s16:
5943 case NEON::BI__builtin_neon_vqdmlslh_s16: {
5944 SmallVector<Value *, 2> ProductOps;
5945 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[1]));
5946 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[2]));
5947 auto *VTy = llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: 4);
5948 Ops[1] = EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmull, Tys: VTy),
5949 Ops&: ProductOps, name: "vqdmlXl");
5950 Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
5951 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: CI, Name: "lane0");
5952
5953 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
5954 ? Intrinsic::aarch64_neon_sqadd
5955 : Intrinsic::aarch64_neon_sqsub;
5956 // Drop the 2nd multiplication argument before the accumulation
5957 Ops.pop_back();
5958 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccumInt, Tys: Int32Ty), Ops, name: "vqdmlXl");
5959 }
5960 case NEON::BI__builtin_neon_vqshlud_n_s64: {
5961 Ops[1] = Builder.CreateZExt(V: Ops[1], DestTy: Int64Ty);
5962 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqshlu, Tys: Int64Ty),
5963 Ops, name: "vqshlu_n");
5964 }
5965 case NEON::BI__builtin_neon_vqshld_n_u64:
5966 case NEON::BI__builtin_neon_vqshld_n_s64: {
5967 Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
5968 ? Intrinsic::aarch64_neon_uqshl
5969 : Intrinsic::aarch64_neon_sqshl;
5970 Ops[1] = Builder.CreateZExt(V: Ops[1], DestTy: Int64Ty);
5971 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Int64Ty), Ops, name: "vqshl_n");
5972 }
5973 case NEON::BI__builtin_neon_vrshrd_n_u64:
5974 case NEON::BI__builtin_neon_vrshrd_n_s64: {
5975 Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
5976 ? Intrinsic::aarch64_neon_urshl
5977 : Intrinsic::aarch64_neon_srshl;
5978 int SV = cast<ConstantInt>(Val: Ops[1])->getSExtValue();
5979 Ops[1] = ConstantInt::get(Ty: Int64Ty, V: -SV);
5980 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Int64Ty), Ops, name: "vrshr_n");
5981 }
5982 case NEON::BI__builtin_neon_vrsrad_n_u64:
5983 case NEON::BI__builtin_neon_vrsrad_n_s64: {
5984 Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
5985 ? Intrinsic::aarch64_neon_urshl
5986 : Intrinsic::aarch64_neon_srshl;
5987 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty);
5988 Ops[2] = Builder.CreateNeg(V: Ops[2]);
5989 Ops[1] = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Int, Tys: Int64Ty),
5990 Args: {Ops[1], Builder.CreateSExt(V: Ops[2], DestTy: Int64Ty)});
5991 return Builder.CreateAdd(LHS: Ops[0], RHS: Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty));
5992 }
5993 case NEON::BI__builtin_neon_vshld_n_s64:
5994 case NEON::BI__builtin_neon_vshld_n_u64: {
5995 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[1]);
5996 return Builder.CreateShl(
5997 LHS: Ops[0], RHS: ConstantInt::get(Ty: Int64Ty, V: Amt->getZExtValue()), Name: "shld_n");
5998 }
5999 case NEON::BI__builtin_neon_vshrd_n_s64: {
6000 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[1]);
6001 return Builder.CreateAShr(
6002 LHS: Ops[0], RHS: ConstantInt::get(Ty: Int64Ty, V: std::min(a: static_cast<uint64_t>(63),
6003 b: Amt->getZExtValue())),
6004 Name: "shrd_n");
6005 }
6006 case NEON::BI__builtin_neon_vshrd_n_u64: {
6007 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[1]);
6008 uint64_t ShiftAmt = Amt->getZExtValue();
6009 // Right-shifting an unsigned value by its size yields 0.
6010 if (ShiftAmt == 64)
6011 return ConstantInt::get(Ty: Int64Ty, V: 0);
6012 return Builder.CreateLShr(LHS: Ops[0], RHS: ConstantInt::get(Ty: Int64Ty, V: ShiftAmt),
6013 Name: "shrd_n");
6014 }
6015 case NEON::BI__builtin_neon_vsrad_n_s64: {
6016 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[2]);
6017 Ops[1] = Builder.CreateAShr(
6018 LHS: Ops[1], RHS: ConstantInt::get(Ty: Int64Ty, V: std::min(a: static_cast<uint64_t>(63),
6019 b: Amt->getZExtValue())),
6020 Name: "shrd_n");
6021 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
6022 }
6023 case NEON::BI__builtin_neon_vsrad_n_u64: {
6024 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[2]);
6025 uint64_t ShiftAmt = Amt->getZExtValue();
6026 // Right-shifting an unsigned value by its size yields 0.
6027 // As Op + 0 = Op, return Ops[0] directly.
6028 if (ShiftAmt == 64)
6029 return Ops[0];
6030 Ops[1] = Builder.CreateLShr(LHS: Ops[1], RHS: ConstantInt::get(Ty: Int64Ty, V: ShiftAmt),
6031 Name: "shrd_n");
6032 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
6033 }
6034 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
6035 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
6036 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
6037 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
6038 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "lane");
6039 SmallVector<Value *, 2> ProductOps;
6040 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[1]));
6041 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[2]));
6042 auto *VTy = llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: 4);
6043 Ops[1] = EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmull, Tys: VTy),
6044 Ops&: ProductOps, name: "vqdmlXl");
6045 Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
6046 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: CI, Name: "lane0");
6047 // Drop lane-selection and the corresponding vector argument (these have
6048 // already been used)
6049 Ops.pop_back_n(NumItems: 2);
6050
6051 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
6052 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
6053 ? Intrinsic::aarch64_neon_sqadd
6054 : Intrinsic::aarch64_neon_sqsub;
6055 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccInt, Tys: Int32Ty), Ops, name: "vqdmlXl");
6056 }
6057 case NEON::BI__builtin_neon_vqdmlals_s32:
6058 case NEON::BI__builtin_neon_vqdmlsls_s32: {
6059 SmallVector<Value *, 2> ProductOps;
6060 ProductOps.push_back(Elt: Ops[1]);
6061 ProductOps.push_back(Elt: Ops[2]);
6062 Ops[1] =
6063 EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmulls_scalar),
6064 Ops&: ProductOps, name: "vqdmlXl");
6065
6066 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
6067 ? Intrinsic::aarch64_neon_sqadd
6068 : Intrinsic::aarch64_neon_sqsub;
6069 // Drop the 2nd multiplication argument before the accumulation
6070 Ops.pop_back();
6071 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccumInt, Tys: Int64Ty), Ops, name: "vqdmlXl");
6072 }
6073 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
6074 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
6075 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
6076 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
6077 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "lane");
6078 SmallVector<Value *, 2> ProductOps;
6079 ProductOps.push_back(Elt: Ops[1]);
6080 ProductOps.push_back(Elt: Ops[2]);
6081 Ops[1] =
6082 EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmulls_scalar),
6083 Ops&: ProductOps, name: "vqdmlXl");
6084 // Drop lane-selection and the corresponding vector argument (these have
6085 // already been used)
6086 Ops.pop_back_n(NumItems: 2);
6087
6088 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
6089 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
6090 ? Intrinsic::aarch64_neon_sqadd
6091 : Intrinsic::aarch64_neon_sqsub;
6092 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccInt, Tys: Int64Ty), Ops, name: "vqdmlXl");
6093 }
6094 case NEON::BI__builtin_neon_vget_lane_bf16:
6095 case NEON::BI__builtin_neon_vduph_lane_bf16:
6096 case NEON::BI__builtin_neon_vduph_lane_f16: {
6097 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
6098 }
6099 case NEON::BI__builtin_neon_vgetq_lane_bf16:
6100 case NEON::BI__builtin_neon_vduph_laneq_bf16:
6101 case NEON::BI__builtin_neon_vduph_laneq_f16: {
6102 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
6103 }
6104 case NEON::BI__builtin_neon_vcvt_bf16_f32: {
6105 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6106 llvm::Type *V4BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 4);
6107 return Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F32), DestTy: V4BF16);
6108 }
6109 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32: {
6110 SmallVector<int, 16> ConcatMask(8);
6111 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
6112 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6113 llvm::Type *V4BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 4);
6114 llvm::Value *Trunc =
6115 Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F32), DestTy: V4BF16);
6116 return Builder.CreateShuffleVector(
6117 V1: Trunc, V2: ConstantAggregateZero::get(Ty: V4BF16), Mask: ConcatMask);
6118 }
6119 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32: {
6120 SmallVector<int, 16> ConcatMask(8);
6121 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
6122 SmallVector<int, 16> LoMask(4);
6123 std::iota(first: LoMask.begin(), last: LoMask.end(), value: 0);
6124 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6125 llvm::Type *V4BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 4);
6126 llvm::Type *V8BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 8);
6127 llvm::Value *Inactive = Builder.CreateShuffleVector(
6128 V: Builder.CreateBitCast(V: Ops[0], DestTy: V8BF16), Mask: LoMask);
6129 llvm::Value *Trunc =
6130 Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[1], DestTy: V4F32), DestTy: V4BF16);
6131 return Builder.CreateShuffleVector(V1: Inactive, V2: Trunc, Mask: ConcatMask);
6132 }
6133 case NEON::BI__builtin_neon_vcvt_f16_f32: {
6134 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6135 llvm::Type *V4F16 = FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 4);
6136 return Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F32), DestTy: V4F16);
6137 }
6138 case NEON::BI__builtin_neon_vcvt_f32_f16: {
6139 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6140 llvm::Type *V4F16 = FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 4);
6141 return Builder.CreateFPExt(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F16), DestTy: V4F32);
6142 }
6143
6144 case clang::AArch64::BI_InterlockedAdd:
6145 case clang::AArch64::BI_InterlockedAdd_acq:
6146 case clang::AArch64::BI_InterlockedAdd_rel:
6147 case clang::AArch64::BI_InterlockedAdd_nf:
6148 case clang::AArch64::BI_InterlockedAdd64:
6149 case clang::AArch64::BI_InterlockedAdd64_acq:
6150 case clang::AArch64::BI_InterlockedAdd64_rel:
6151 case clang::AArch64::BI_InterlockedAdd64_nf: {
6152 Address DestAddr = CheckAtomicAlignment(CGF&: *this, E);
6153 Value *Val = Ops[1];
6154 llvm::AtomicOrdering Ordering;
6155 switch (BuiltinID) {
6156 case clang::AArch64::BI_InterlockedAdd:
6157 case clang::AArch64::BI_InterlockedAdd64:
6158 Ordering = llvm::AtomicOrdering::SequentiallyConsistent;
6159 break;
6160 case clang::AArch64::BI_InterlockedAdd_acq:
6161 case clang::AArch64::BI_InterlockedAdd64_acq:
6162 Ordering = llvm::AtomicOrdering::Acquire;
6163 break;
6164 case clang::AArch64::BI_InterlockedAdd_rel:
6165 case clang::AArch64::BI_InterlockedAdd64_rel:
6166 Ordering = llvm::AtomicOrdering::Release;
6167 break;
6168 case clang::AArch64::BI_InterlockedAdd_nf:
6169 case clang::AArch64::BI_InterlockedAdd64_nf:
6170 Ordering = llvm::AtomicOrdering::Monotonic;
6171 break;
6172 default:
6173 llvm_unreachable("missing builtin ID in switch!");
6174 }
6175 AtomicRMWInst *RMWI =
6176 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Addr: DestAddr, Val, Ordering);
6177 return Builder.CreateAdd(LHS: RMWI, RHS: Val);
6178 }
6179 }
6180
6181 llvm::FixedVectorType *VTy = GetNeonType(CGF: this, TypeFlags: Type);
6182 llvm::Type *Ty = VTy;
6183 if (!Ty)
6184 return nullptr;
6185
6186 bool ExtractLow = false;
6187 bool ExtendLaneArg = false;
6188 switch (BuiltinID) {
6189 default: return nullptr;
6190 case NEON::BI__builtin_neon_vbsl_v:
6191 case NEON::BI__builtin_neon_vbslq_v: {
6192 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
6193 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: BitTy, Name: "vbsl");
6194 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: BitTy, Name: "vbsl");
6195 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: BitTy, Name: "vbsl");
6196
6197 Ops[1] = Builder.CreateAnd(LHS: Ops[0], RHS: Ops[1], Name: "vbsl");
6198 Ops[2] = Builder.CreateAnd(LHS: Builder.CreateNot(V: Ops[0]), RHS: Ops[2], Name: "vbsl");
6199 Ops[0] = Builder.CreateOr(LHS: Ops[1], RHS: Ops[2], Name: "vbsl");
6200 return Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6201 }
6202 case NEON::BI__builtin_neon_vfma_lane_v:
6203 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
6204 // The ARM builtins (and instructions) have the addend as the first
6205 // operand, but the 'fma' intrinsics have it last. Swap it around here.
6206 Value *Addend = Ops[0];
6207 Value *Multiplicand = Ops[1];
6208 Value *LaneSource = Ops[2];
6209 Ops[0] = Multiplicand;
6210 Ops[1] = LaneSource;
6211 Ops[2] = Addend;
6212
6213 // Now adjust things to handle the lane access.
6214 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
6215 ? llvm::FixedVectorType::get(ElementType: VTy->getElementType(),
6216 NumElts: VTy->getNumElements() / 2)
6217 : VTy;
6218 llvm::Constant *cst = cast<Constant>(Val: Ops[3]);
6219 Value *SV = llvm::ConstantVector::getSplat(EC: VTy->getElementCount(), Elt: cst);
6220 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: SourceTy);
6221 Ops[1] = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[1], Mask: SV, Name: "lane");
6222
6223 Ops.pop_back();
6224 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
6225 : Intrinsic::fma;
6226 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "fmla");
6227 }
6228 case NEON::BI__builtin_neon_vfma_laneq_v: {
6229 auto *VTy = cast<llvm::FixedVectorType>(Val: Ty);
6230 // v1f64 fma should be mapped to Neon scalar f64 fma
6231 if (VTy && VTy->getElementType() == DoubleTy) {
6232 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
6233 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: DoubleTy);
6234 llvm::FixedVectorType *VTy =
6235 GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float64, false, true));
6236 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: VTy);
6237 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "extract");
6238 Value *Result;
6239 Result = emitCallMaybeConstrainedFPBuiltin(
6240 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma,
6241 Ty: DoubleTy, Args: {Ops[1], Ops[2], Ops[0]});
6242 return Builder.CreateBitCast(V: Result, DestTy: Ty);
6243 }
6244 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6245 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6246
6247 auto *STy = llvm::FixedVectorType::get(ElementType: VTy->getElementType(),
6248 NumElts: VTy->getNumElements() * 2);
6249 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: STy);
6250 Value *SV = llvm::ConstantVector::getSplat(EC: VTy->getElementCount(),
6251 Elt: cast<ConstantInt>(Val: Ops[3]));
6252 Ops[2] = Builder.CreateShuffleVector(V1: Ops[2], V2: Ops[2], Mask: SV, Name: "lane");
6253
6254 return emitCallMaybeConstrainedFPBuiltin(
6255 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
6256 Args: {Ops[2], Ops[1], Ops[0]});
6257 }
6258 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
6259 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6260 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6261
6262 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6263 Ops[2] = EmitNeonSplat(V: Ops[2], C: cast<ConstantInt>(Val: Ops[3]));
6264 return emitCallMaybeConstrainedFPBuiltin(
6265 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
6266 Args: {Ops[2], Ops[1], Ops[0]});
6267 }
6268 case NEON::BI__builtin_neon_vfmah_lane_f16:
6269 case NEON::BI__builtin_neon_vfmas_lane_f32:
6270 case NEON::BI__builtin_neon_vfmah_laneq_f16:
6271 case NEON::BI__builtin_neon_vfmas_laneq_f32:
6272 case NEON::BI__builtin_neon_vfmad_lane_f64:
6273 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
6274 llvm::Type *Ty = ConvertType(T: E->getCallReturnType(Ctx: getContext()));
6275 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "extract");
6276 return emitCallMaybeConstrainedFPBuiltin(
6277 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
6278 Args: {Ops[1], Ops[2], Ops[0]});
6279 }
6280 case NEON::BI__builtin_neon_vmull_v:
6281 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6282 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
6283 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
6284 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmull");
6285 case NEON::BI__builtin_neon_vmax_v:
6286 case NEON::BI__builtin_neon_vmaxq_v:
6287 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6288 Int = usgn ? Intrinsic::umax : Intrinsic::smax;
6289 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
6290 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmax");
6291 case NEON::BI__builtin_neon_vmaxh_f16: {
6292 Int = Intrinsic::aarch64_neon_fmax;
6293 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmax");
6294 }
6295 case NEON::BI__builtin_neon_vmin_v:
6296 case NEON::BI__builtin_neon_vminq_v:
6297 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6298 Int = usgn ? Intrinsic::umin : Intrinsic::smin;
6299 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
6300 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmin");
6301 case NEON::BI__builtin_neon_vminh_f16: {
6302 Int = Intrinsic::aarch64_neon_fmin;
6303 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmin");
6304 }
6305 case NEON::BI__builtin_neon_vabd_v:
6306 case NEON::BI__builtin_neon_vabdq_v:
6307 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6308 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
6309 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
6310 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vabd");
6311 case NEON::BI__builtin_neon_vpadal_v:
6312 case NEON::BI__builtin_neon_vpadalq_v: {
6313 unsigned ArgElts = VTy->getNumElements();
6314 llvm::IntegerType *EltTy = cast<IntegerType>(Val: VTy->getElementType());
6315 unsigned BitWidth = EltTy->getBitWidth();
6316 auto *ArgTy = llvm::FixedVectorType::get(
6317 ElementType: llvm::IntegerType::get(C&: getLLVMContext(), NumBits: BitWidth / 2), NumElts: 2 * ArgElts);
6318 llvm::Type* Tys[2] = { VTy, ArgTy };
6319 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
6320 SmallVector<llvm::Value*, 1> TmpOps;
6321 TmpOps.push_back(Elt: Ops[1]);
6322 Function *F = CGM.getIntrinsic(IID: Int, Tys);
6323 llvm::Value *tmp = EmitNeonCall(F, Ops&: TmpOps, name: "vpadal");
6324 llvm::Value *addend = Builder.CreateBitCast(V: Ops[0], DestTy: tmp->getType());
6325 return Builder.CreateAdd(LHS: tmp, RHS: addend);
6326 }
6327 case NEON::BI__builtin_neon_vpmin_v:
6328 case NEON::BI__builtin_neon_vpminq_v:
6329 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6330 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
6331 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
6332 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpmin");
6333 case NEON::BI__builtin_neon_vpmax_v:
6334 case NEON::BI__builtin_neon_vpmaxq_v:
6335 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6336 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
6337 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
6338 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpmax");
6339 case NEON::BI__builtin_neon_vminnm_v:
6340 case NEON::BI__builtin_neon_vminnmq_v:
6341 Int = Intrinsic::aarch64_neon_fminnm;
6342 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vminnm");
6343 case NEON::BI__builtin_neon_vminnmh_f16:
6344 Int = Intrinsic::aarch64_neon_fminnm;
6345 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vminnm");
6346 case NEON::BI__builtin_neon_vmaxnm_v:
6347 case NEON::BI__builtin_neon_vmaxnmq_v:
6348 Int = Intrinsic::aarch64_neon_fmaxnm;
6349 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmaxnm");
6350 case NEON::BI__builtin_neon_vmaxnmh_f16:
6351 Int = Intrinsic::aarch64_neon_fmaxnm;
6352 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmaxnm");
6353 case NEON::BI__builtin_neon_vrecpss_f32: {
6354 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_frecps, Tys: FloatTy),
6355 Ops, name: "vrecps");
6356 }
6357 case NEON::BI__builtin_neon_vrecpsd_f64:
6358 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_frecps, Tys: DoubleTy),
6359 Ops, name: "vrecps");
6360 case NEON::BI__builtin_neon_vrecpsh_f16:
6361 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_frecps, Tys: HalfTy),
6362 Ops, name: "vrecps");
6363 case NEON::BI__builtin_neon_vqshrun_n_v:
6364 Int = Intrinsic::aarch64_neon_sqshrun;
6365 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshrun_n");
6366 case NEON::BI__builtin_neon_vqrshrun_n_v:
6367 Int = Intrinsic::aarch64_neon_sqrshrun;
6368 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqrshrun_n");
6369 case NEON::BI__builtin_neon_vqshrn_n_v:
6370 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
6371 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshrn_n");
6372 case NEON::BI__builtin_neon_vrshrn_n_v:
6373 Int = Intrinsic::aarch64_neon_rshrn;
6374 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrshrn_n");
6375 case NEON::BI__builtin_neon_vqrshrn_n_v:
6376 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
6377 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqrshrn_n");
6378 case NEON::BI__builtin_neon_vrndah_f16: {
6379 Int = Builder.getIsFPConstrained()
6380 ? Intrinsic::experimental_constrained_round
6381 : Intrinsic::round;
6382 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrnda");
6383 }
6384 case NEON::BI__builtin_neon_vrnda_v:
6385 case NEON::BI__builtin_neon_vrndaq_v: {
6386 Int = Builder.getIsFPConstrained()
6387 ? Intrinsic::experimental_constrained_round
6388 : Intrinsic::round;
6389 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrnda");
6390 }
6391 case NEON::BI__builtin_neon_vrndih_f16: {
6392 Int = Builder.getIsFPConstrained()
6393 ? Intrinsic::experimental_constrained_nearbyint
6394 : Intrinsic::nearbyint;
6395 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndi");
6396 }
6397 case NEON::BI__builtin_neon_vrndmh_f16: {
6398 Int = Builder.getIsFPConstrained()
6399 ? Intrinsic::experimental_constrained_floor
6400 : Intrinsic::floor;
6401 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndm");
6402 }
6403 case NEON::BI__builtin_neon_vrndm_v:
6404 case NEON::BI__builtin_neon_vrndmq_v: {
6405 Int = Builder.getIsFPConstrained()
6406 ? Intrinsic::experimental_constrained_floor
6407 : Intrinsic::floor;
6408 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndm");
6409 }
6410 case NEON::BI__builtin_neon_vrndnh_f16: {
6411 Int = Builder.getIsFPConstrained()
6412 ? Intrinsic::experimental_constrained_roundeven
6413 : Intrinsic::roundeven;
6414 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndn");
6415 }
6416 case NEON::BI__builtin_neon_vrndn_v:
6417 case NEON::BI__builtin_neon_vrndnq_v: {
6418 Int = Builder.getIsFPConstrained()
6419 ? Intrinsic::experimental_constrained_roundeven
6420 : Intrinsic::roundeven;
6421 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndn");
6422 }
6423 case NEON::BI__builtin_neon_vrndns_f32: {
6424 Int = Builder.getIsFPConstrained()
6425 ? Intrinsic::experimental_constrained_roundeven
6426 : Intrinsic::roundeven;
6427 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: FloatTy), Ops, name: "vrndn");
6428 }
6429 case NEON::BI__builtin_neon_vrndph_f16: {
6430 Int = Builder.getIsFPConstrained()
6431 ? Intrinsic::experimental_constrained_ceil
6432 : Intrinsic::ceil;
6433 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndp");
6434 }
6435 case NEON::BI__builtin_neon_vrndp_v:
6436 case NEON::BI__builtin_neon_vrndpq_v: {
6437 Int = Builder.getIsFPConstrained()
6438 ? Intrinsic::experimental_constrained_ceil
6439 : Intrinsic::ceil;
6440 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndp");
6441 }
6442 case NEON::BI__builtin_neon_vrndxh_f16: {
6443 Int = Builder.getIsFPConstrained()
6444 ? Intrinsic::experimental_constrained_rint
6445 : Intrinsic::rint;
6446 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndx");
6447 }
6448 case NEON::BI__builtin_neon_vrndx_v:
6449 case NEON::BI__builtin_neon_vrndxq_v: {
6450 Int = Builder.getIsFPConstrained()
6451 ? Intrinsic::experimental_constrained_rint
6452 : Intrinsic::rint;
6453 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndx");
6454 }
6455 case NEON::BI__builtin_neon_vrndh_f16: {
6456 Int = Builder.getIsFPConstrained()
6457 ? Intrinsic::experimental_constrained_trunc
6458 : Intrinsic::trunc;
6459 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndz");
6460 }
6461 case NEON::BI__builtin_neon_vrnd_v:
6462 case NEON::BI__builtin_neon_vrndq_v: {
6463 Int = Builder.getIsFPConstrained()
6464 ? Intrinsic::experimental_constrained_trunc
6465 : Intrinsic::trunc;
6466 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndz");
6467 }
6468 case NEON::BI__builtin_neon_vcvt_f64_v:
6469 case NEON::BI__builtin_neon_vcvtq_f64_v:
6470 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6471 Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
6472 return usgn ? Builder.CreateUIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt")
6473 : Builder.CreateSIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt");
6474 case NEON::BI__builtin_neon_vcvt_f64_f32: {
6475 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&
6476 "unexpected vcvt_f64_f32 builtin");
6477 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
6478 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: GetNeonType(CGF: this, TypeFlags: SrcFlag));
6479
6480 return Builder.CreateFPExt(V: Ops[0], DestTy: Ty, Name: "vcvt");
6481 }
6482 case NEON::BI__builtin_neon_vcvt_f32_f64: {
6483 assert(Type.getEltType() == NeonTypeFlags::Float32 &&
6484 "unexpected vcvt_f32_f64 builtin");
6485 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
6486 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: GetNeonType(CGF: this, TypeFlags: SrcFlag));
6487
6488 return Builder.CreateFPTrunc(V: Ops[0], DestTy: Ty, Name: "vcvt");
6489 }
6490 case NEON::BI__builtin_neon_vcvta_s16_f16:
6491 case NEON::BI__builtin_neon_vcvta_u16_f16:
6492 case NEON::BI__builtin_neon_vcvta_s32_v:
6493 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
6494 case NEON::BI__builtin_neon_vcvtaq_s32_v:
6495 case NEON::BI__builtin_neon_vcvta_u32_v:
6496 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
6497 case NEON::BI__builtin_neon_vcvtaq_u32_v:
6498 case NEON::BI__builtin_neon_vcvta_s64_v:
6499 case NEON::BI__builtin_neon_vcvtaq_s64_v:
6500 case NEON::BI__builtin_neon_vcvta_u64_v:
6501 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
6502 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
6503 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6504 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvta");
6505 }
6506 case NEON::BI__builtin_neon_vcvtm_s16_f16:
6507 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
6508 case NEON::BI__builtin_neon_vcvtm_u16_f16:
6509 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
6510 case NEON::BI__builtin_neon_vcvtm_s32_v:
6511 case NEON::BI__builtin_neon_vcvtmq_s32_v:
6512 case NEON::BI__builtin_neon_vcvtm_u32_v:
6513 case NEON::BI__builtin_neon_vcvtmq_u32_v:
6514 case NEON::BI__builtin_neon_vcvtm_s64_v:
6515 case NEON::BI__builtin_neon_vcvtmq_s64_v:
6516 case NEON::BI__builtin_neon_vcvtm_u64_v:
6517 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
6518 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
6519 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6520 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtm");
6521 }
6522 case NEON::BI__builtin_neon_vcvtn_s16_f16:
6523 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
6524 case NEON::BI__builtin_neon_vcvtn_u16_f16:
6525 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
6526 case NEON::BI__builtin_neon_vcvtn_s32_v:
6527 case NEON::BI__builtin_neon_vcvtnq_s32_v:
6528 case NEON::BI__builtin_neon_vcvtn_u32_v:
6529 case NEON::BI__builtin_neon_vcvtnq_u32_v:
6530 case NEON::BI__builtin_neon_vcvtn_s64_v:
6531 case NEON::BI__builtin_neon_vcvtnq_s64_v:
6532 case NEON::BI__builtin_neon_vcvtn_u64_v:
6533 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
6534 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
6535 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6536 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtn");
6537 }
6538 case NEON::BI__builtin_neon_vcvtp_s16_f16:
6539 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
6540 case NEON::BI__builtin_neon_vcvtp_u16_f16:
6541 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
6542 case NEON::BI__builtin_neon_vcvtp_s32_v:
6543 case NEON::BI__builtin_neon_vcvtpq_s32_v:
6544 case NEON::BI__builtin_neon_vcvtp_u32_v:
6545 case NEON::BI__builtin_neon_vcvtpq_u32_v:
6546 case NEON::BI__builtin_neon_vcvtp_s64_v:
6547 case NEON::BI__builtin_neon_vcvtpq_s64_v:
6548 case NEON::BI__builtin_neon_vcvtp_u64_v:
6549 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
6550 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
6551 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6552 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtp");
6553 }
6554 case NEON::BI__builtin_neon_vmulx_v:
6555 case NEON::BI__builtin_neon_vmulxq_v: {
6556 Int = Intrinsic::aarch64_neon_fmulx;
6557 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmulx");
6558 }
6559 case NEON::BI__builtin_neon_vmulxh_lane_f16:
6560 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
6561 // vmulx_lane should be mapped to Neon scalar mulx after
6562 // extracting the scalar element
6563 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2], Name: "extract");
6564 Ops.pop_back();
6565 Int = Intrinsic::aarch64_neon_fmulx;
6566 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmulx");
6567 }
6568 case NEON::BI__builtin_neon_vmul_lane_v:
6569 case NEON::BI__builtin_neon_vmul_laneq_v: {
6570 // v1f64 vmul_lane should be mapped to Neon scalar mul lane
6571 bool Quad = false;
6572 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
6573 Quad = true;
6574 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
6575 llvm::FixedVectorType *VTy =
6576 GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
6577 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: VTy);
6578 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2], Name: "extract");
6579 Value *Result = Builder.CreateFMul(L: Ops[0], R: Ops[1]);
6580 return Builder.CreateBitCast(V: Result, DestTy: Ty);
6581 }
6582 case NEON::BI__builtin_neon_vpmaxnm_v:
6583 case NEON::BI__builtin_neon_vpmaxnmq_v: {
6584 Int = Intrinsic::aarch64_neon_fmaxnmp;
6585 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpmaxnm");
6586 }
6587 case NEON::BI__builtin_neon_vpminnm_v:
6588 case NEON::BI__builtin_neon_vpminnmq_v: {
6589 Int = Intrinsic::aarch64_neon_fminnmp;
6590 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpminnm");
6591 }
6592 case NEON::BI__builtin_neon_vsqrth_f16: {
6593 Int = Builder.getIsFPConstrained()
6594 ? Intrinsic::experimental_constrained_sqrt
6595 : Intrinsic::sqrt;
6596 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vsqrt");
6597 }
6598 case NEON::BI__builtin_neon_vsqrt_v:
6599 case NEON::BI__builtin_neon_vsqrtq_v: {
6600 Int = Builder.getIsFPConstrained()
6601 ? Intrinsic::experimental_constrained_sqrt
6602 : Intrinsic::sqrt;
6603 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6604 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vsqrt");
6605 }
6606 case NEON::BI__builtin_neon_vrbit_v:
6607 case NEON::BI__builtin_neon_vrbitq_v: {
6608 Int = Intrinsic::bitreverse;
6609 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrbit");
6610 }
6611 case NEON::BI__builtin_neon_vmaxv_f16: {
6612 Int = Intrinsic::aarch64_neon_fmaxv;
6613 Ty = HalfTy;
6614 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6615 llvm::Type *Tys[2] = {Ty, VTy};
6616 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxv");
6617 }
6618 case NEON::BI__builtin_neon_vmaxvq_f16: {
6619 Int = Intrinsic::aarch64_neon_fmaxv;
6620 Ty = HalfTy;
6621 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6622 llvm::Type *Tys[2] = {Ty, VTy};
6623 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxv");
6624 }
6625 case NEON::BI__builtin_neon_vminv_f16: {
6626 Int = Intrinsic::aarch64_neon_fminv;
6627 Ty = HalfTy;
6628 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6629 llvm::Type *Tys[2] = {Ty, VTy};
6630 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminv");
6631 }
6632 case NEON::BI__builtin_neon_vminvq_f16: {
6633 Int = Intrinsic::aarch64_neon_fminv;
6634 Ty = HalfTy;
6635 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6636 llvm::Type *Tys[2] = {Ty, VTy};
6637 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminv");
6638 }
6639 case NEON::BI__builtin_neon_vmaxnmv_f16: {
6640 Int = Intrinsic::aarch64_neon_fmaxnmv;
6641 Ty = HalfTy;
6642 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6643 llvm::Type *Tys[2] = {Ty, VTy};
6644 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxnmv");
6645 }
6646 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
6647 Int = Intrinsic::aarch64_neon_fmaxnmv;
6648 Ty = HalfTy;
6649 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6650 llvm::Type *Tys[2] = {Ty, VTy};
6651 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxnmv");
6652 }
6653 case NEON::BI__builtin_neon_vminnmv_f16: {
6654 Int = Intrinsic::aarch64_neon_fminnmv;
6655 Ty = HalfTy;
6656 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6657 llvm::Type *Tys[2] = {Ty, VTy};
6658 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminnmv");
6659 }
6660 case NEON::BI__builtin_neon_vminnmvq_f16: {
6661 Int = Intrinsic::aarch64_neon_fminnmv;
6662 Ty = HalfTy;
6663 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6664 llvm::Type *Tys[2] = {Ty, VTy};
6665 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminnmv");
6666 }
6667 case NEON::BI__builtin_neon_vmul_n_f64: {
6668 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
6669 Value *RHS = Builder.CreateBitCast(V: Ops[1], DestTy: DoubleTy);
6670 return Builder.CreateFMul(L: Ops[0], R: RHS);
6671 }
6672 case NEON::BI__builtin_neon_vaddlv_u8:
6673 case NEON::BI__builtin_neon_vaddlvq_u8:
6674 case NEON::BI__builtin_neon_vaddlv_u16:
6675 case NEON::BI__builtin_neon_vaddlvq_u16: {
6676 Int = Intrinsic::aarch64_neon_uaddlv;
6677 Ty = Int32Ty;
6678 VTy = cast<llvm::FixedVectorType>(Val: Ops[0]->getType());
6679 llvm::Type *Tys[2] = {Ty, VTy};
6680 Value *Result = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vaddlv");
6681 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6682 return Builder.CreateTrunc(V: Result, DestTy: Int16Ty);
6683 return Result;
6684 }
6685 case NEON::BI__builtin_neon_vaddlv_s8:
6686 case NEON::BI__builtin_neon_vaddlvq_s8:
6687 case NEON::BI__builtin_neon_vaddlv_s16:
6688 case NEON::BI__builtin_neon_vaddlvq_s16: {
6689 Int = Intrinsic::aarch64_neon_saddlv;
6690 Ty = Int32Ty;
6691 VTy = cast<llvm::FixedVectorType>(Val: Ops[0]->getType());
6692 llvm::Type *Tys[2] = {Ty, VTy};
6693 Value *Result = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vaddlv");
6694 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6695 return Builder.CreateTrunc(V: Result, DestTy: Int16Ty);
6696 return Result;
6697 }
6698 case NEON::BI__builtin_neon_vsri_n_v:
6699 case NEON::BI__builtin_neon_vsriq_n_v: {
6700 Int = Intrinsic::aarch64_neon_vsri;
6701 llvm::Function *Intrin = CGM.getIntrinsic(IID: Int, Tys: Ty);
6702 return EmitNeonCall(F: Intrin, Ops, name: "vsri_n");
6703 }
6704 case NEON::BI__builtin_neon_vsli_n_v:
6705 case NEON::BI__builtin_neon_vsliq_n_v: {
6706 Int = Intrinsic::aarch64_neon_vsli;
6707 llvm::Function *Intrin = CGM.getIntrinsic(IID: Int, Tys: Ty);
6708 return EmitNeonCall(F: Intrin, Ops, name: "vsli_n");
6709 }
6710 case NEON::BI__builtin_neon_vsra_n_v:
6711 case NEON::BI__builtin_neon_vsraq_n_v:
6712 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6713 Ops[1] = EmitNeonRShiftImm(Vec: Ops[1], Shift: Ops[2], Ty, usgn, name: "vsra_n");
6714 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
6715 case NEON::BI__builtin_neon_vrsra_n_v:
6716 case NEON::BI__builtin_neon_vrsraq_n_v: {
6717 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
6718 SmallVector<llvm::Value*,2> TmpOps;
6719 TmpOps.push_back(Elt: Ops[1]);
6720 TmpOps.push_back(Elt: Ops[2]);
6721 Function* F = CGM.getIntrinsic(IID: Int, Tys: Ty);
6722 llvm::Value *tmp = EmitNeonCall(F, Ops&: TmpOps, name: "vrshr_n", shift: 1, rightshift: true);
6723 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: VTy);
6724 return Builder.CreateAdd(LHS: Ops[0], RHS: tmp);
6725 }
6726 case NEON::BI__builtin_neon_vld1_v:
6727 case NEON::BI__builtin_neon_vld1q_v: {
6728 return Builder.CreateAlignedLoad(Ty: VTy, Addr: Ops[0], Align: PtrOp0.getAlignment());
6729 }
6730 case NEON::BI__builtin_neon_vst1_v:
6731 case NEON::BI__builtin_neon_vst1q_v:
6732 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: VTy);
6733 return Builder.CreateAlignedStore(Val: Ops[1], Addr: Ops[0], Align: PtrOp0.getAlignment());
6734 case NEON::BI__builtin_neon_vld1_lane_v:
6735 case NEON::BI__builtin_neon_vld1q_lane_v: {
6736 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6737 Ops[0] = Builder.CreateAlignedLoad(Ty: VTy->getElementType(), Addr: Ops[0],
6738 Align: PtrOp0.getAlignment());
6739 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vld1_lane");
6740 }
6741 case NEON::BI__builtin_neon_vldap1_lane_s64:
6742 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
6743 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6744 llvm::LoadInst *LI = Builder.CreateAlignedLoad(
6745 Ty: VTy->getElementType(), Addr: Ops[0], Align: PtrOp0.getAlignment());
6746 LI->setAtomic(Ordering: llvm::AtomicOrdering::Acquire);
6747 Ops[0] = LI;
6748 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vldap1_lane");
6749 }
6750 case NEON::BI__builtin_neon_vld1_dup_v:
6751 case NEON::BI__builtin_neon_vld1q_dup_v: {
6752 Value *V = PoisonValue::get(T: Ty);
6753 Ops[0] = Builder.CreateAlignedLoad(Ty: VTy->getElementType(), Addr: Ops[0],
6754 Align: PtrOp0.getAlignment());
6755 llvm::Constant *CI = ConstantInt::get(Ty: Int32Ty, V: 0);
6756 Ops[0] = Builder.CreateInsertElement(Vec: V, NewElt: Ops[0], Idx: CI);
6757 return EmitNeonSplat(V: Ops[0], C: CI);
6758 }
6759 case NEON::BI__builtin_neon_vst1_lane_v:
6760 case NEON::BI__builtin_neon_vst1q_lane_v:
6761 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6762 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2]);
6763 return Builder.CreateAlignedStore(Val: Ops[1], Addr: Ops[0], Align: PtrOp0.getAlignment());
6764 case NEON::BI__builtin_neon_vstl1_lane_s64:
6765 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
6766 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6767 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2]);
6768 llvm::StoreInst *SI =
6769 Builder.CreateAlignedStore(Val: Ops[1], Addr: Ops[0], Align: PtrOp0.getAlignment());
6770 SI->setAtomic(Ordering: llvm::AtomicOrdering::Release);
6771 return SI;
6772 }
6773 case NEON::BI__builtin_neon_vld2_v:
6774 case NEON::BI__builtin_neon_vld2q_v: {
6775 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6776 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld2, Tys);
6777 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld2");
6778 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6779 }
6780 case NEON::BI__builtin_neon_vld3_v:
6781 case NEON::BI__builtin_neon_vld3q_v: {
6782 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6783 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld3, Tys);
6784 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld3");
6785 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6786 }
6787 case NEON::BI__builtin_neon_vld4_v:
6788 case NEON::BI__builtin_neon_vld4q_v: {
6789 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6790 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld4, Tys);
6791 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld4");
6792 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6793 }
6794 case NEON::BI__builtin_neon_vld2_dup_v:
6795 case NEON::BI__builtin_neon_vld2q_dup_v: {
6796 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6797 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld2r, Tys);
6798 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld2");
6799 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6800 }
6801 case NEON::BI__builtin_neon_vld3_dup_v:
6802 case NEON::BI__builtin_neon_vld3q_dup_v: {
6803 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6804 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld3r, Tys);
6805 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld3");
6806 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6807 }
6808 case NEON::BI__builtin_neon_vld4_dup_v:
6809 case NEON::BI__builtin_neon_vld4q_dup_v: {
6810 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6811 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld4r, Tys);
6812 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld4");
6813 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6814 }
6815 case NEON::BI__builtin_neon_vld2_lane_v:
6816 case NEON::BI__builtin_neon_vld2q_lane_v: {
6817 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6818 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld2lane, Tys);
6819 std::rotate(first: Ops.begin() + 1, middle: Ops.begin() + 2, last: Ops.end());
6820 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6821 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6822 Ops[3] = Builder.CreateZExt(V: Ops[3], DestTy: Int64Ty);
6823 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: "vld2_lane");
6824 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6825 }
6826 case NEON::BI__builtin_neon_vld3_lane_v:
6827 case NEON::BI__builtin_neon_vld3q_lane_v: {
6828 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6829 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld3lane, Tys);
6830 std::rotate(first: Ops.begin() + 1, middle: Ops.begin() + 2, last: Ops.end());
6831 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6832 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6833 Ops[3] = Builder.CreateBitCast(V: Ops[3], DestTy: Ty);
6834 Ops[4] = Builder.CreateZExt(V: Ops[4], DestTy: Int64Ty);
6835 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: "vld3_lane");
6836 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6837 }
6838 case NEON::BI__builtin_neon_vld4_lane_v:
6839 case NEON::BI__builtin_neon_vld4q_lane_v: {
6840 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6841 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld4lane, Tys);
6842 std::rotate(first: Ops.begin() + 1, middle: Ops.begin() + 2, last: Ops.end());
6843 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6844 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6845 Ops[3] = Builder.CreateBitCast(V: Ops[3], DestTy: Ty);
6846 Ops[4] = Builder.CreateBitCast(V: Ops[4], DestTy: Ty);
6847 Ops[5] = Builder.CreateZExt(V: Ops[5], DestTy: Int64Ty);
6848 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: "vld4_lane");
6849 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6850 }
6851 case NEON::BI__builtin_neon_vst2_v:
6852 case NEON::BI__builtin_neon_vst2q_v: {
6853 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6854 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
6855 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st2, Tys),
6856 Ops, name: "");
6857 }
6858 case NEON::BI__builtin_neon_vst2_lane_v:
6859 case NEON::BI__builtin_neon_vst2q_lane_v: {
6860 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6861 Ops[2] = Builder.CreateZExt(V: Ops[2], DestTy: Int64Ty);
6862 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6863 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st2lane, Tys),
6864 Ops, name: "");
6865 }
6866 case NEON::BI__builtin_neon_vst3_v:
6867 case NEON::BI__builtin_neon_vst3q_v: {
6868 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6869 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6870 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st3, Tys),
6871 Ops, name: "");
6872 }
6873 case NEON::BI__builtin_neon_vst3_lane_v:
6874 case NEON::BI__builtin_neon_vst3q_lane_v: {
6875 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6876 Ops[3] = Builder.CreateZExt(V: Ops[3], DestTy: Int64Ty);
6877 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6878 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st3lane, Tys),
6879 Ops, name: "");
6880 }
6881 case NEON::BI__builtin_neon_vst4_v:
6882 case NEON::BI__builtin_neon_vst4q_v: {
6883 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6884 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6885 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st4, Tys),
6886 Ops, name: "");
6887 }
6888 case NEON::BI__builtin_neon_vst4_lane_v:
6889 case NEON::BI__builtin_neon_vst4q_lane_v: {
6890 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6891 Ops[4] = Builder.CreateZExt(V: Ops[4], DestTy: Int64Ty);
6892 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
6893 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st4lane, Tys),
6894 Ops, name: "");
6895 }
6896 case NEON::BI__builtin_neon_vtrn_v:
6897 case NEON::BI__builtin_neon_vtrnq_v: {
6898 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6899 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6900 Value *SV = nullptr;
6901
6902 for (unsigned vi = 0; vi != 2; ++vi) {
6903 SmallVector<int, 16> Indices;
6904 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6905 Indices.push_back(Elt: i+vi);
6906 Indices.push_back(Elt: i+e+vi);
6907 }
6908 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
6909 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vtrn");
6910 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
6911 }
6912 return SV;
6913 }
6914 case NEON::BI__builtin_neon_vuzp_v:
6915 case NEON::BI__builtin_neon_vuzpq_v: {
6916 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6917 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6918 Value *SV = nullptr;
6919
6920 for (unsigned vi = 0; vi != 2; ++vi) {
6921 SmallVector<int, 16> Indices;
6922 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
6923 Indices.push_back(Elt: 2*i+vi);
6924
6925 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
6926 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vuzp");
6927 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
6928 }
6929 return SV;
6930 }
6931 case NEON::BI__builtin_neon_vzip_v:
6932 case NEON::BI__builtin_neon_vzipq_v: {
6933 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6934 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6935 Value *SV = nullptr;
6936
6937 for (unsigned vi = 0; vi != 2; ++vi) {
6938 SmallVector<int, 16> Indices;
6939 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6940 Indices.push_back(Elt: (i + vi*e) >> 1);
6941 Indices.push_back(Elt: ((i + vi*e) >> 1)+e);
6942 }
6943 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
6944 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vzip");
6945 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
6946 }
6947 return SV;
6948 }
6949 case NEON::BI__builtin_neon_vqtbl1q_v: {
6950 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl1, Tys: Ty),
6951 Ops, name: "vtbl1");
6952 }
6953 case NEON::BI__builtin_neon_vqtbl2q_v: {
6954 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl2, Tys: Ty),
6955 Ops, name: "vtbl2");
6956 }
6957 case NEON::BI__builtin_neon_vqtbl3q_v: {
6958 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl3, Tys: Ty),
6959 Ops, name: "vtbl3");
6960 }
6961 case NEON::BI__builtin_neon_vqtbl4q_v: {
6962 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl4, Tys: Ty),
6963 Ops, name: "vtbl4");
6964 }
6965 case NEON::BI__builtin_neon_vqtbx1q_v: {
6966 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx1, Tys: Ty),
6967 Ops, name: "vtbx1");
6968 }
6969 case NEON::BI__builtin_neon_vqtbx2q_v: {
6970 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx2, Tys: Ty),
6971 Ops, name: "vtbx2");
6972 }
6973 case NEON::BI__builtin_neon_vqtbx3q_v: {
6974 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx3, Tys: Ty),
6975 Ops, name: "vtbx3");
6976 }
6977 case NEON::BI__builtin_neon_vqtbx4q_v: {
6978 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx4, Tys: Ty),
6979 Ops, name: "vtbx4");
6980 }
6981 case NEON::BI__builtin_neon_vsqadd_v:
6982 case NEON::BI__builtin_neon_vsqaddq_v: {
6983 Int = Intrinsic::aarch64_neon_usqadd;
6984 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vsqadd");
6985 }
6986 case NEON::BI__builtin_neon_vuqadd_v:
6987 case NEON::BI__builtin_neon_vuqaddq_v: {
6988 Int = Intrinsic::aarch64_neon_suqadd;
6989 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vuqadd");
6990 }
6991
6992 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
6993 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
6994 case NEON::BI__builtin_neon_vluti2_laneq_f16:
6995 case NEON::BI__builtin_neon_vluti2_laneq_p16:
6996 case NEON::BI__builtin_neon_vluti2_laneq_p8:
6997 case NEON::BI__builtin_neon_vluti2_laneq_s16:
6998 case NEON::BI__builtin_neon_vluti2_laneq_s8:
6999 case NEON::BI__builtin_neon_vluti2_laneq_u16:
7000 case NEON::BI__builtin_neon_vluti2_laneq_u8: {
7001 Int = Intrinsic::aarch64_neon_vluti2_laneq;
7002 llvm::Type *Tys[2];
7003 Tys[0] = Ty;
7004 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
7005 /*isQuad*/ false));
7006 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_laneq");
7007 }
7008 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
7009 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
7010 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
7011 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
7012 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
7013 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
7014 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
7015 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
7016 case NEON::BI__builtin_neon_vluti2q_laneq_u8: {
7017 Int = Intrinsic::aarch64_neon_vluti2_laneq;
7018 llvm::Type *Tys[2];
7019 Tys[0] = Ty;
7020 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
7021 /*isQuad*/ true));
7022 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_laneq");
7023 }
7024 case NEON::BI__builtin_neon_vluti2_lane_mf8:
7025 case NEON::BI__builtin_neon_vluti2_lane_bf16:
7026 case NEON::BI__builtin_neon_vluti2_lane_f16:
7027 case NEON::BI__builtin_neon_vluti2_lane_p16:
7028 case NEON::BI__builtin_neon_vluti2_lane_p8:
7029 case NEON::BI__builtin_neon_vluti2_lane_s16:
7030 case NEON::BI__builtin_neon_vluti2_lane_s8:
7031 case NEON::BI__builtin_neon_vluti2_lane_u16:
7032 case NEON::BI__builtin_neon_vluti2_lane_u8: {
7033 Int = Intrinsic::aarch64_neon_vluti2_lane;
7034 llvm::Type *Tys[2];
7035 Tys[0] = Ty;
7036 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
7037 /*isQuad*/ false));
7038 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_lane");
7039 }
7040 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
7041 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
7042 case NEON::BI__builtin_neon_vluti2q_lane_f16:
7043 case NEON::BI__builtin_neon_vluti2q_lane_p16:
7044 case NEON::BI__builtin_neon_vluti2q_lane_p8:
7045 case NEON::BI__builtin_neon_vluti2q_lane_s16:
7046 case NEON::BI__builtin_neon_vluti2q_lane_s8:
7047 case NEON::BI__builtin_neon_vluti2q_lane_u16:
7048 case NEON::BI__builtin_neon_vluti2q_lane_u8: {
7049 Int = Intrinsic::aarch64_neon_vluti2_lane;
7050 llvm::Type *Tys[2];
7051 Tys[0] = Ty;
7052 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
7053 /*isQuad*/ true));
7054 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_lane");
7055 }
7056 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
7057 case NEON::BI__builtin_neon_vluti4q_lane_p8:
7058 case NEON::BI__builtin_neon_vluti4q_lane_s8:
7059 case NEON::BI__builtin_neon_vluti4q_lane_u8: {
7060 Int = Intrinsic::aarch64_neon_vluti4q_lane;
7061 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_lane");
7062 }
7063 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
7064 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
7065 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
7066 case NEON::BI__builtin_neon_vluti4q_laneq_u8: {
7067 Int = Intrinsic::aarch64_neon_vluti4q_laneq;
7068 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_laneq");
7069 }
7070 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
7071 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
7072 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
7073 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
7074 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: {
7075 Int = Intrinsic::aarch64_neon_vluti4q_lane_x2;
7076 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_lane_x2");
7077 }
7078 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
7079 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
7080 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
7081 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
7082 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: {
7083 Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
7084 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_laneq_x2");
7085 }
7086 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
7087 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fmmla,
7088 Tys: {llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8),
7089 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16)},
7090 Ops, E, name: "fmmla");
7091 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
7092 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fmmla,
7093 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4),
7094 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16)},
7095 Ops, E, name: "fmmla");
7096 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
7097 ExtractLow = true;
7098 [[fallthrough]];
7099 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
7100 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
7101 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl1,
7102 Ty0: llvm::FixedVectorType::get(ElementType: BFloatTy, NumElts: 8),
7103 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt1");
7104 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
7105 ExtractLow = true;
7106 [[fallthrough]];
7107 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
7108 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
7109 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl2,
7110 Ty0: llvm::FixedVectorType::get(ElementType: BFloatTy, NumElts: 8),
7111 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt2");
7112 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
7113 ExtractLow = true;
7114 [[fallthrough]];
7115 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
7116 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
7117 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl1,
7118 Ty0: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8),
7119 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt1");
7120 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
7121 ExtractLow = true;
7122 [[fallthrough]];
7123 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
7124 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
7125 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl2,
7126 Ty0: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8),
7127 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt2");
7128 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
7129 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn,
7130 Ty0: llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 8),
7131 Ty1: Ops[0]->getType(), Extract: false, Ops, E, name: "vfcvtn");
7132 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
7133 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn,
7134 Ty0: llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 8),
7135 Ty1: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4), Extract: false, Ops,
7136 E, name: "vfcvtn");
7137 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
7138 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn,
7139 Ty0: llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16),
7140 Ty1: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8), Extract: false, Ops,
7141 E, name: "vfcvtn");
7142 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: {
7143 llvm::Type *Ty = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16);
7144 Ops[0] = Builder.CreateInsertVector(DstType: Ty, SrcVec: PoisonValue::get(T: Ty), SubVec: Ops[0],
7145 Idx: uint64_t(0));
7146 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn2, Ty0: Ty,
7147 Ty1: Ops[1]->getType(), Extract: false, Ops, E, name: "vfcvtn2");
7148 }
7149
7150 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
7151 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
7152 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot2, ExtendLaneArg: false, RetTy: HalfTy,
7153 Ops, E, name: "fdot2");
7154 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
7155 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
7156 ExtendLaneArg = true;
7157 [[fallthrough]];
7158 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
7159 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
7160 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot2_lane,
7161 ExtendLaneArg, RetTy: HalfTy, Ops, E, name: "fdot2_lane");
7162 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
7163 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
7164 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot4, ExtendLaneArg: false,
7165 RetTy: FloatTy, Ops, E, name: "fdot4");
7166 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
7167 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
7168 ExtendLaneArg = true;
7169 [[fallthrough]];
7170 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
7171 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
7172 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot4_lane,
7173 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "fdot4_lane");
7174
7175 case NEON::BI__builtin_neon_vdot_f32_f16:
7176 case NEON::BI__builtin_neon_vdotq_f32_f16: {
7177 llvm::Type *InputTy =
7178 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
7179 llvm::Type *Tys[2] = {Ty, InputTy};
7180 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_fdot, Tys),
7181 Ops, name: "vdot");
7182 }
7183
7184 case NEON::BI__builtin_neon_vdot_lane_f32_f16:
7185 case NEON::BI__builtin_neon_vdot_laneq_f32_f16:
7186 case NEON::BI__builtin_neon_vdotq_lane_f32_f16:
7187 case NEON::BI__builtin_neon_vdotq_laneq_f32_f16: {
7188 llvm::FixedVectorType *InputTy =
7189 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
7190 llvm::FixedVectorType *LaneTy = llvm::FixedVectorType::get(
7191 ElementType: HalfTy, NumElts: Ops[2]->getType()->getPrimitiveSizeInBits() / 16);
7192 // Treat the lane argument as a splat and use non-lane version of the
7193 // intrinsic.
7194 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: LaneTy);
7195 Ops[2] = EmitNeonSplat(V: Ops[2], C: cast<ConstantInt>(Val: Ops[3]),
7196 Count: InputTy->getElementCount());
7197 llvm::Type *Tys[2] = {Ty, InputTy};
7198 Ops.pop_back();
7199 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_fdot, Tys),
7200 Ops, name: "vdot");
7201 }
7202
7203 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
7204 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalb,
7205 Tys: {llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8)}, Ops, E,
7206 name: "vmlal");
7207 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
7208 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalt,
7209 Tys: {llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8)}, Ops, E,
7210 name: "vmlal");
7211 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
7212 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlallbb,
7213 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7214 name: "vmlall");
7215 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
7216 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlallbt,
7217 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7218 name: "vmlall");
7219 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
7220 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalltb,
7221 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7222 name: "vmlall");
7223 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
7224 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalltt,
7225 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7226 name: "vmlall");
7227 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
7228 ExtendLaneArg = true;
7229 [[fallthrough]];
7230 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
7231 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalb_lane,
7232 ExtendLaneArg, RetTy: HalfTy, Ops, E, name: "vmlal_lane");
7233 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
7234 ExtendLaneArg = true;
7235 [[fallthrough]];
7236 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
7237 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalt_lane,
7238 ExtendLaneArg, RetTy: HalfTy, Ops, E, name: "vmlal_lane");
7239 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
7240 ExtendLaneArg = true;
7241 [[fallthrough]];
7242 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
7243 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlallbb_lane,
7244 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7245 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
7246 ExtendLaneArg = true;
7247 [[fallthrough]];
7248 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
7249 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlallbt_lane,
7250 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7251 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
7252 ExtendLaneArg = true;
7253 [[fallthrough]];
7254 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
7255 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalltb_lane,
7256 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7257 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
7258 ExtendLaneArg = true;
7259 [[fallthrough]];
7260 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
7261 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalltt_lane,
7262 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7263 case NEON::BI__builtin_neon_vamin_f16:
7264 case NEON::BI__builtin_neon_vaminq_f16:
7265 case NEON::BI__builtin_neon_vamin_f32:
7266 case NEON::BI__builtin_neon_vaminq_f32:
7267 case NEON::BI__builtin_neon_vaminq_f64: {
7268 Int = Intrinsic::aarch64_neon_famin;
7269 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "famin");
7270 }
7271 case NEON::BI__builtin_neon_vamax_f16:
7272 case NEON::BI__builtin_neon_vamaxq_f16:
7273 case NEON::BI__builtin_neon_vamax_f32:
7274 case NEON::BI__builtin_neon_vamaxq_f32:
7275 case NEON::BI__builtin_neon_vamaxq_f64: {
7276 Int = Intrinsic::aarch64_neon_famax;
7277 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "famax");
7278 }
7279 case NEON::BI__builtin_neon_vscale_f16:
7280 case NEON::BI__builtin_neon_vscaleq_f16:
7281 case NEON::BI__builtin_neon_vscale_f32:
7282 case NEON::BI__builtin_neon_vscaleq_f32:
7283 case NEON::BI__builtin_neon_vscaleq_f64: {
7284 Int = Intrinsic::aarch64_neon_fp8_fscale;
7285 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "fscale");
7286 }
7287 }
7288}
7289
7290Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
7291 const CallExpr *E) {
7292 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
7293 BuiltinID == BPF::BI__builtin_btf_type_id ||
7294 BuiltinID == BPF::BI__builtin_preserve_type_info ||
7295 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
7296 "unexpected BPF builtin");
7297
7298 // A sequence number, injected into IR builtin functions, to
7299 // prevent CSE given the only difference of the function
7300 // may just be the debuginfo metadata.
7301 static uint32_t BuiltinSeqNum;
7302
7303 switch (BuiltinID) {
7304 default:
7305 llvm_unreachable("Unexpected BPF builtin");
7306 case BPF::BI__builtin_preserve_field_info: {
7307 const Expr *Arg = E->getArg(Arg: 0);
7308 bool IsBitField = Arg->IgnoreParens()->getObjectKind() == OK_BitField;
7309
7310 if (!getDebugInfo()) {
7311 CGM.Error(loc: E->getExprLoc(),
7312 error: "using __builtin_preserve_field_info() without -g");
7313 return IsBitField ? EmitLValue(E: Arg).getRawBitFieldPointer(CGF&: *this)
7314 : EmitLValue(E: Arg).emitRawPointer(CGF&: *this);
7315 }
7316
7317 // Enable underlying preserve_*_access_index() generation.
7318 bool OldIsInPreservedAIRegion = IsInPreservedAIRegion;
7319 IsInPreservedAIRegion = true;
7320 Value *FieldAddr = IsBitField ? EmitLValue(E: Arg).getRawBitFieldPointer(CGF&: *this)
7321 : EmitLValue(E: Arg).emitRawPointer(CGF&: *this);
7322 IsInPreservedAIRegion = OldIsInPreservedAIRegion;
7323
7324 ConstantInt *C = cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 1)));
7325 Value *InfoKind = ConstantInt::get(Ty: Int64Ty, V: C->getSExtValue());
7326
7327 // Built the IR for the preserve_field_info intrinsic.
7328 llvm::Function *FnGetFieldInfo = Intrinsic::getOrInsertDeclaration(
7329 M: &CGM.getModule(), id: Intrinsic::bpf_preserve_field_info,
7330 OverloadTys: {FieldAddr->getType()});
7331 return Builder.CreateCall(Callee: FnGetFieldInfo, Args: {FieldAddr, InfoKind});
7332 }
7333 case BPF::BI__builtin_btf_type_id:
7334 case BPF::BI__builtin_preserve_type_info: {
7335 if (!getDebugInfo()) {
7336 CGM.Error(loc: E->getExprLoc(), error: "using builtin function without -g");
7337 return nullptr;
7338 }
7339
7340 const Expr *Arg0 = E->getArg(Arg: 0);
7341 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
7342 Ty: Arg0->getType(), Loc: Arg0->getExprLoc());
7343
7344 ConstantInt *Flag = cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 1)));
7345 Value *FlagValue = ConstantInt::get(Ty: Int64Ty, V: Flag->getSExtValue());
7346 Value *SeqNumVal = ConstantInt::get(Ty: Int32Ty, V: BuiltinSeqNum++);
7347
7348 llvm::Function *FnDecl;
7349 if (BuiltinID == BPF::BI__builtin_btf_type_id)
7350 FnDecl = Intrinsic::getOrInsertDeclaration(
7351 M: &CGM.getModule(), id: Intrinsic::bpf_btf_type_id, OverloadTys: {});
7352 else
7353 FnDecl = Intrinsic::getOrInsertDeclaration(
7354 M: &CGM.getModule(), id: Intrinsic::bpf_preserve_type_info, OverloadTys: {});
7355 CallInst *Fn = Builder.CreateCall(Callee: FnDecl, Args: {SeqNumVal, FlagValue});
7356 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
7357 return Fn;
7358 }
7359 case BPF::BI__builtin_preserve_enum_value: {
7360 if (!getDebugInfo()) {
7361 CGM.Error(loc: E->getExprLoc(), error: "using builtin function without -g");
7362 return nullptr;
7363 }
7364
7365 const Expr *Arg0 = E->getArg(Arg: 0);
7366 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
7367 Ty: Arg0->getType(), Loc: Arg0->getExprLoc());
7368
7369 // Find enumerator
7370 const auto *UO = cast<UnaryOperator>(Val: Arg0->IgnoreParens());
7371 const auto *CE = cast<CStyleCastExpr>(Val: UO->getSubExpr());
7372 const auto *DR = cast<DeclRefExpr>(Val: CE->getSubExpr());
7373 const auto *Enumerator = cast<EnumConstantDecl>(Val: DR->getDecl());
7374
7375 auto InitVal = Enumerator->getInitVal();
7376 std::string InitValStr;
7377 if (InitVal.isNegative() || InitVal > uint64_t(INT64_MAX))
7378 InitValStr = std::to_string(val: InitVal.getSExtValue());
7379 else
7380 InitValStr = std::to_string(val: InitVal.getZExtValue());
7381 std::string EnumStr = Enumerator->getNameAsString() + ":" + InitValStr;
7382 Value *EnumStrVal = Builder.CreateGlobalString(Str: EnumStr);
7383
7384 ConstantInt *Flag = cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 1)));
7385 Value *FlagValue = ConstantInt::get(Ty: Int64Ty, V: Flag->getSExtValue());
7386 Value *SeqNumVal = ConstantInt::get(Ty: Int32Ty, V: BuiltinSeqNum++);
7387
7388 llvm::Function *IntrinsicFn = Intrinsic::getOrInsertDeclaration(
7389 M: &CGM.getModule(), id: Intrinsic::bpf_preserve_enum_value, OverloadTys: {});
7390 CallInst *Fn =
7391 Builder.CreateCall(Callee: IntrinsicFn, Args: {SeqNumVal, EnumStrVal, FlagValue});
7392 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
7393 return Fn;
7394 }
7395 }
7396}
7397
7398llvm::Value *CodeGenFunction::
7399BuildVector(ArrayRef<llvm::Value*> Ops) {
7400 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
7401 "Not a power-of-two sized vector!");
7402 bool AllConstants = true;
7403 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
7404 AllConstants &= isa<Constant>(Val: Ops[i]);
7405
7406 // If this is a constant vector, create a ConstantVector.
7407 if (AllConstants) {
7408 SmallVector<llvm::Constant*, 16> CstOps;
7409 for (llvm::Value *Op : Ops)
7410 CstOps.push_back(Elt: cast<Constant>(Val: Op));
7411 return llvm::ConstantVector::get(V: CstOps);
7412 }
7413
7414 // Otherwise, insertelement the values to build the vector.
7415 Value *Result = llvm::PoisonValue::get(
7416 T: llvm::FixedVectorType::get(ElementType: Ops[0]->getType(), NumElts: Ops.size()));
7417
7418 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
7419 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Ops[i], Idx: Builder.getInt64(C: i));
7420
7421 return Result;
7422}
7423
7424Value *CodeGenFunction::EmitAArch64CpuInit() {
7425 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
7426 llvm::FunctionCallee Func =
7427 CGM.CreateRuntimeFunction(Ty: FTy, Name: "__init_cpu_features_resolver");
7428 cast<llvm::GlobalValue>(Val: Func.getCallee())->setDSOLocal(true);
7429 cast<llvm::GlobalValue>(Val: Func.getCallee())
7430 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
7431 return Builder.CreateCall(Callee: Func);
7432}
7433
7434Value *CodeGenFunction::EmitAArch64CpuSupports(const CallExpr *E) {
7435 const Expr *ArgExpr = E->getArg(Arg: 0)->IgnoreParenCasts();
7436 StringRef ArgStr = cast<StringLiteral>(Val: ArgExpr)->getString();
7437 llvm::SmallVector<StringRef, 8> OrigFeatures;
7438 ArgStr.split(A&: OrigFeatures, Separator: "+");
7439 llvm::SmallVector<StringRef, 8> Features;
7440 for (StringRef Feature : OrigFeatures) {
7441 Feature = Feature.trim();
7442 if (!llvm::AArch64::parseFMVExtension(Extension: Feature))
7443 return Builder.getFalse();
7444 if (Feature != "default")
7445 Features.push_back(Elt: Feature);
7446 }
7447 return EmitAArch64CpuSupports(FeatureStrs: Features);
7448}
7449
7450llvm::Value *
7451CodeGenFunction::EmitAArch64CpuSupports(ArrayRef<StringRef> FeaturesStrs) {
7452 llvm::APInt FeaturesMask = llvm::AArch64::getCpuSupportsMask(Features: FeaturesStrs);
7453 Value *Result = Builder.getTrue();
7454 if (FeaturesMask != 0) {
7455 // Get features from structure in runtime library
7456 // struct {
7457 // unsigned long long features;
7458 // } __aarch64_cpu_features;
7459 llvm::Type *STy = llvm::StructType::get(elt1: Int64Ty);
7460 llvm::Constant *AArch64CPUFeatures =
7461 CGM.CreateRuntimeVariable(Ty: STy, Name: "__aarch64_cpu_features");
7462 cast<llvm::GlobalValue>(Val: AArch64CPUFeatures)->setDSOLocal(true);
7463 llvm::Value *CpuFeatures = Builder.CreateGEP(
7464 Ty: STy, Ptr: AArch64CPUFeatures,
7465 IdxList: {ConstantInt::get(Ty: Int32Ty, V: 0), ConstantInt::get(Ty: Int32Ty, V: 0)});
7466 Value *Features = Builder.CreateAlignedLoad(Ty: Int64Ty, Addr: CpuFeatures,
7467 Align: CharUnits::fromQuantity(Quantity: 8));
7468 Value *Mask = Builder.getInt(AI: FeaturesMask.trunc(width: 64));
7469 Value *Bitset = Builder.CreateAnd(LHS: Features, RHS: Mask);
7470 Value *Cmp = Builder.CreateICmpEQ(LHS: Bitset, RHS: Mask);
7471 Result = Builder.CreateAnd(LHS: Result, RHS: Cmp);
7472 }
7473 return Result;
7474}
7475