1//===---------- ARM.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfo.h"
14#include "CGBuiltin.h"
15#include "CGDebugInfo.h"
16#include "TargetInfo.h"
17#include "clang/Basic/AArch64CodeGenUtils.h"
18#include "clang/Basic/TargetBuiltins.h"
19#include "clang/CodeGenUtils/TargetUtils.h"
20#include "llvm/IR/InlineAsm.h"
21#include "llvm/IR/IntrinsicsAArch64.h"
22#include "llvm/IR/IntrinsicsARM.h"
23#include "llvm/IR/IntrinsicsBPF.h"
24#include "llvm/Support/AArch64MemoryHints.h"
25#include "llvm/TargetParser/AArch64TargetParser.h"
26
27#include <numeric>
28
29using namespace clang;
30using namespace CodeGen;
31using namespace llvm;
32using namespace clang::aarch64;
33
34static std::optional<CodeGenFunction::MSVCIntrin>
35translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
36 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
37 switch (BuiltinID) {
38 default:
39 return std::nullopt;
40 case clang::AArch64::BI_BitScanForward:
41 case clang::AArch64::BI_BitScanForward64:
42 return MSVCIntrin::_BitScanForward;
43 case clang::AArch64::BI_BitScanReverse:
44 case clang::AArch64::BI_BitScanReverse64:
45 return MSVCIntrin::_BitScanReverse;
46 case clang::AArch64::BI_InterlockedAnd64:
47 return MSVCIntrin::_InterlockedAnd;
48 case clang::AArch64::BI_InterlockedExchange64:
49 return MSVCIntrin::_InterlockedExchange;
50 case clang::AArch64::BI_InterlockedExchangeAdd64:
51 return MSVCIntrin::_InterlockedExchangeAdd;
52 case clang::AArch64::BI_InterlockedExchangeSub64:
53 return MSVCIntrin::_InterlockedExchangeSub;
54 case clang::AArch64::BI_InterlockedOr64:
55 return MSVCIntrin::_InterlockedOr;
56 case clang::AArch64::BI_InterlockedXor64:
57 return MSVCIntrin::_InterlockedXor;
58 case clang::AArch64::BI_InterlockedDecrement64:
59 return MSVCIntrin::_InterlockedDecrement;
60 case clang::AArch64::BI_InterlockedIncrement64:
61 return MSVCIntrin::_InterlockedIncrement;
62 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
63 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
64 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
65 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
66 return MSVCIntrin::_InterlockedExchangeAdd_acq;
67 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
68 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
69 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
70 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
71 return MSVCIntrin::_InterlockedExchangeAdd_rel;
72 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
73 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
74 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
75 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
76 return MSVCIntrin::_InterlockedExchangeAdd_nf;
77 case clang::AArch64::BI_InterlockedExchange8_acq:
78 case clang::AArch64::BI_InterlockedExchange16_acq:
79 case clang::AArch64::BI_InterlockedExchange_acq:
80 case clang::AArch64::BI_InterlockedExchange64_acq:
81 case clang::AArch64::BI_InterlockedExchangePointer_acq:
82 return MSVCIntrin::_InterlockedExchange_acq;
83 case clang::AArch64::BI_InterlockedExchange8_rel:
84 case clang::AArch64::BI_InterlockedExchange16_rel:
85 case clang::AArch64::BI_InterlockedExchange_rel:
86 case clang::AArch64::BI_InterlockedExchange64_rel:
87 case clang::AArch64::BI_InterlockedExchangePointer_rel:
88 return MSVCIntrin::_InterlockedExchange_rel;
89 case clang::AArch64::BI_InterlockedExchange8_nf:
90 case clang::AArch64::BI_InterlockedExchange16_nf:
91 case clang::AArch64::BI_InterlockedExchange_nf:
92 case clang::AArch64::BI_InterlockedExchange64_nf:
93 case clang::AArch64::BI_InterlockedExchangePointer_nf:
94 return MSVCIntrin::_InterlockedExchange_nf;
95 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
96 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
97 case clang::AArch64::BI_InterlockedCompareExchange_acq:
98 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
99 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
100 return MSVCIntrin::_InterlockedCompareExchange_acq;
101 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
102 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
103 case clang::AArch64::BI_InterlockedCompareExchange_rel:
104 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
105 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
106 return MSVCIntrin::_InterlockedCompareExchange_rel;
107 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
108 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
109 case clang::AArch64::BI_InterlockedCompareExchange_nf:
110 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
111 return MSVCIntrin::_InterlockedCompareExchange_nf;
112 case clang::AArch64::BI_InterlockedCompareExchange128:
113 return MSVCIntrin::_InterlockedCompareExchange128;
114 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
115 return MSVCIntrin::_InterlockedCompareExchange128_acq;
116 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
117 return MSVCIntrin::_InterlockedCompareExchange128_nf;
118 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
119 return MSVCIntrin::_InterlockedCompareExchange128_rel;
120 case clang::AArch64::BI_InterlockedOr8_acq:
121 case clang::AArch64::BI_InterlockedOr16_acq:
122 case clang::AArch64::BI_InterlockedOr_acq:
123 case clang::AArch64::BI_InterlockedOr64_acq:
124 return MSVCIntrin::_InterlockedOr_acq;
125 case clang::AArch64::BI_InterlockedOr8_rel:
126 case clang::AArch64::BI_InterlockedOr16_rel:
127 case clang::AArch64::BI_InterlockedOr_rel:
128 case clang::AArch64::BI_InterlockedOr64_rel:
129 return MSVCIntrin::_InterlockedOr_rel;
130 case clang::AArch64::BI_InterlockedOr8_nf:
131 case clang::AArch64::BI_InterlockedOr16_nf:
132 case clang::AArch64::BI_InterlockedOr_nf:
133 case clang::AArch64::BI_InterlockedOr64_nf:
134 return MSVCIntrin::_InterlockedOr_nf;
135 case clang::AArch64::BI_InterlockedXor8_acq:
136 case clang::AArch64::BI_InterlockedXor16_acq:
137 case clang::AArch64::BI_InterlockedXor_acq:
138 case clang::AArch64::BI_InterlockedXor64_acq:
139 return MSVCIntrin::_InterlockedXor_acq;
140 case clang::AArch64::BI_InterlockedXor8_rel:
141 case clang::AArch64::BI_InterlockedXor16_rel:
142 case clang::AArch64::BI_InterlockedXor_rel:
143 case clang::AArch64::BI_InterlockedXor64_rel:
144 return MSVCIntrin::_InterlockedXor_rel;
145 case clang::AArch64::BI_InterlockedXor8_nf:
146 case clang::AArch64::BI_InterlockedXor16_nf:
147 case clang::AArch64::BI_InterlockedXor_nf:
148 case clang::AArch64::BI_InterlockedXor64_nf:
149 return MSVCIntrin::_InterlockedXor_nf;
150 case clang::AArch64::BI_InterlockedAnd8_acq:
151 case clang::AArch64::BI_InterlockedAnd16_acq:
152 case clang::AArch64::BI_InterlockedAnd_acq:
153 case clang::AArch64::BI_InterlockedAnd64_acq:
154 return MSVCIntrin::_InterlockedAnd_acq;
155 case clang::AArch64::BI_InterlockedAnd8_rel:
156 case clang::AArch64::BI_InterlockedAnd16_rel:
157 case clang::AArch64::BI_InterlockedAnd_rel:
158 case clang::AArch64::BI_InterlockedAnd64_rel:
159 return MSVCIntrin::_InterlockedAnd_rel;
160 case clang::AArch64::BI_InterlockedAnd8_nf:
161 case clang::AArch64::BI_InterlockedAnd16_nf:
162 case clang::AArch64::BI_InterlockedAnd_nf:
163 case clang::AArch64::BI_InterlockedAnd64_nf:
164 return MSVCIntrin::_InterlockedAnd_nf;
165 case clang::AArch64::BI_InterlockedIncrement16_acq:
166 case clang::AArch64::BI_InterlockedIncrement_acq:
167 case clang::AArch64::BI_InterlockedIncrement64_acq:
168 return MSVCIntrin::_InterlockedIncrement_acq;
169 case clang::AArch64::BI_InterlockedIncrement16_rel:
170 case clang::AArch64::BI_InterlockedIncrement_rel:
171 case clang::AArch64::BI_InterlockedIncrement64_rel:
172 return MSVCIntrin::_InterlockedIncrement_rel;
173 case clang::AArch64::BI_InterlockedIncrement16_nf:
174 case clang::AArch64::BI_InterlockedIncrement_nf:
175 case clang::AArch64::BI_InterlockedIncrement64_nf:
176 return MSVCIntrin::_InterlockedIncrement_nf;
177 case clang::AArch64::BI_InterlockedDecrement16_acq:
178 case clang::AArch64::BI_InterlockedDecrement_acq:
179 case clang::AArch64::BI_InterlockedDecrement64_acq:
180 return MSVCIntrin::_InterlockedDecrement_acq;
181 case clang::AArch64::BI_InterlockedDecrement16_rel:
182 case clang::AArch64::BI_InterlockedDecrement_rel:
183 case clang::AArch64::BI_InterlockedDecrement64_rel:
184 return MSVCIntrin::_InterlockedDecrement_rel;
185 case clang::AArch64::BI_InterlockedDecrement16_nf:
186 case clang::AArch64::BI_InterlockedDecrement_nf:
187 case clang::AArch64::BI_InterlockedDecrement64_nf:
188 return MSVCIntrin::_InterlockedDecrement_nf;
189 case clang::AArch64::BI_ReadWriteBarrier:
190 return MSVCIntrin::_ReadWriteBarrier;
191 }
192 llvm_unreachable("must return from switch");
193}
194
195static std::optional<CodeGenFunction::MSVCIntrin>
196translateArmToMsvcIntrin(unsigned BuiltinID) {
197 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
198 switch (BuiltinID) {
199 default:
200 return std::nullopt;
201 case clang::ARM::BI_BitScanForward:
202 case clang::ARM::BI_BitScanForward64:
203 return MSVCIntrin::_BitScanForward;
204 case clang::ARM::BI_BitScanReverse:
205 case clang::ARM::BI_BitScanReverse64:
206 return MSVCIntrin::_BitScanReverse;
207 case clang::ARM::BI_InterlockedAnd64:
208 return MSVCIntrin::_InterlockedAnd;
209 case clang::ARM::BI_InterlockedExchange64:
210 return MSVCIntrin::_InterlockedExchange;
211 case clang::ARM::BI_InterlockedExchangeAdd64:
212 return MSVCIntrin::_InterlockedExchangeAdd;
213 case clang::ARM::BI_InterlockedExchangeSub64:
214 return MSVCIntrin::_InterlockedExchangeSub;
215 case clang::ARM::BI_InterlockedOr64:
216 return MSVCIntrin::_InterlockedOr;
217 case clang::ARM::BI_InterlockedXor64:
218 return MSVCIntrin::_InterlockedXor;
219 case clang::ARM::BI_InterlockedDecrement64:
220 return MSVCIntrin::_InterlockedDecrement;
221 case clang::ARM::BI_InterlockedIncrement64:
222 return MSVCIntrin::_InterlockedIncrement;
223 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
224 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
225 case clang::ARM::BI_InterlockedExchangeAdd_acq:
226 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
227 return MSVCIntrin::_InterlockedExchangeAdd_acq;
228 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
229 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
230 case clang::ARM::BI_InterlockedExchangeAdd_rel:
231 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
232 return MSVCIntrin::_InterlockedExchangeAdd_rel;
233 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
234 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
235 case clang::ARM::BI_InterlockedExchangeAdd_nf:
236 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
237 return MSVCIntrin::_InterlockedExchangeAdd_nf;
238 case clang::ARM::BI_InterlockedExchange8_acq:
239 case clang::ARM::BI_InterlockedExchange16_acq:
240 case clang::ARM::BI_InterlockedExchange_acq:
241 case clang::ARM::BI_InterlockedExchange64_acq:
242 case clang::ARM::BI_InterlockedExchangePointer_acq:
243 return MSVCIntrin::_InterlockedExchange_acq;
244 case clang::ARM::BI_InterlockedExchange8_rel:
245 case clang::ARM::BI_InterlockedExchange16_rel:
246 case clang::ARM::BI_InterlockedExchange_rel:
247 case clang::ARM::BI_InterlockedExchange64_rel:
248 case clang::ARM::BI_InterlockedExchangePointer_rel:
249 return MSVCIntrin::_InterlockedExchange_rel;
250 case clang::ARM::BI_InterlockedExchange8_nf:
251 case clang::ARM::BI_InterlockedExchange16_nf:
252 case clang::ARM::BI_InterlockedExchange_nf:
253 case clang::ARM::BI_InterlockedExchange64_nf:
254 case clang::ARM::BI_InterlockedExchangePointer_nf:
255 return MSVCIntrin::_InterlockedExchange_nf;
256 case clang::ARM::BI_InterlockedCompareExchange8_acq:
257 case clang::ARM::BI_InterlockedCompareExchange16_acq:
258 case clang::ARM::BI_InterlockedCompareExchange_acq:
259 case clang::ARM::BI_InterlockedCompareExchange64_acq:
260 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
261 return MSVCIntrin::_InterlockedCompareExchange_acq;
262 case clang::ARM::BI_InterlockedCompareExchange8_rel:
263 case clang::ARM::BI_InterlockedCompareExchange16_rel:
264 case clang::ARM::BI_InterlockedCompareExchange_rel:
265 case clang::ARM::BI_InterlockedCompareExchange64_rel:
266 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
267 return MSVCIntrin::_InterlockedCompareExchange_rel;
268 case clang::ARM::BI_InterlockedCompareExchange8_nf:
269 case clang::ARM::BI_InterlockedCompareExchange16_nf:
270 case clang::ARM::BI_InterlockedCompareExchange_nf:
271 case clang::ARM::BI_InterlockedCompareExchange64_nf:
272 return MSVCIntrin::_InterlockedCompareExchange_nf;
273 case clang::ARM::BI_InterlockedOr8_acq:
274 case clang::ARM::BI_InterlockedOr16_acq:
275 case clang::ARM::BI_InterlockedOr_acq:
276 case clang::ARM::BI_InterlockedOr64_acq:
277 return MSVCIntrin::_InterlockedOr_acq;
278 case clang::ARM::BI_InterlockedOr8_rel:
279 case clang::ARM::BI_InterlockedOr16_rel:
280 case clang::ARM::BI_InterlockedOr_rel:
281 case clang::ARM::BI_InterlockedOr64_rel:
282 return MSVCIntrin::_InterlockedOr_rel;
283 case clang::ARM::BI_InterlockedOr8_nf:
284 case clang::ARM::BI_InterlockedOr16_nf:
285 case clang::ARM::BI_InterlockedOr_nf:
286 case clang::ARM::BI_InterlockedOr64_nf:
287 return MSVCIntrin::_InterlockedOr_nf;
288 case clang::ARM::BI_InterlockedXor8_acq:
289 case clang::ARM::BI_InterlockedXor16_acq:
290 case clang::ARM::BI_InterlockedXor_acq:
291 case clang::ARM::BI_InterlockedXor64_acq:
292 return MSVCIntrin::_InterlockedXor_acq;
293 case clang::ARM::BI_InterlockedXor8_rel:
294 case clang::ARM::BI_InterlockedXor16_rel:
295 case clang::ARM::BI_InterlockedXor_rel:
296 case clang::ARM::BI_InterlockedXor64_rel:
297 return MSVCIntrin::_InterlockedXor_rel;
298 case clang::ARM::BI_InterlockedXor8_nf:
299 case clang::ARM::BI_InterlockedXor16_nf:
300 case clang::ARM::BI_InterlockedXor_nf:
301 case clang::ARM::BI_InterlockedXor64_nf:
302 return MSVCIntrin::_InterlockedXor_nf;
303 case clang::ARM::BI_InterlockedAnd8_acq:
304 case clang::ARM::BI_InterlockedAnd16_acq:
305 case clang::ARM::BI_InterlockedAnd_acq:
306 case clang::ARM::BI_InterlockedAnd64_acq:
307 return MSVCIntrin::_InterlockedAnd_acq;
308 case clang::ARM::BI_InterlockedAnd8_rel:
309 case clang::ARM::BI_InterlockedAnd16_rel:
310 case clang::ARM::BI_InterlockedAnd_rel:
311 case clang::ARM::BI_InterlockedAnd64_rel:
312 return MSVCIntrin::_InterlockedAnd_rel;
313 case clang::ARM::BI_InterlockedAnd8_nf:
314 case clang::ARM::BI_InterlockedAnd16_nf:
315 case clang::ARM::BI_InterlockedAnd_nf:
316 case clang::ARM::BI_InterlockedAnd64_nf:
317 return MSVCIntrin::_InterlockedAnd_nf;
318 case clang::ARM::BI_InterlockedIncrement16_acq:
319 case clang::ARM::BI_InterlockedIncrement_acq:
320 case clang::ARM::BI_InterlockedIncrement64_acq:
321 return MSVCIntrin::_InterlockedIncrement_acq;
322 case clang::ARM::BI_InterlockedIncrement16_rel:
323 case clang::ARM::BI_InterlockedIncrement_rel:
324 case clang::ARM::BI_InterlockedIncrement64_rel:
325 return MSVCIntrin::_InterlockedIncrement_rel;
326 case clang::ARM::BI_InterlockedIncrement16_nf:
327 case clang::ARM::BI_InterlockedIncrement_nf:
328 case clang::ARM::BI_InterlockedIncrement64_nf:
329 return MSVCIntrin::_InterlockedIncrement_nf;
330 case clang::ARM::BI_InterlockedDecrement16_acq:
331 case clang::ARM::BI_InterlockedDecrement_acq:
332 case clang::ARM::BI_InterlockedDecrement64_acq:
333 return MSVCIntrin::_InterlockedDecrement_acq;
334 case clang::ARM::BI_InterlockedDecrement16_rel:
335 case clang::ARM::BI_InterlockedDecrement_rel:
336 case clang::ARM::BI_InterlockedDecrement64_rel:
337 return MSVCIntrin::_InterlockedDecrement_rel;
338 case clang::ARM::BI_InterlockedDecrement16_nf:
339 case clang::ARM::BI_InterlockedDecrement_nf:
340 case clang::ARM::BI_InterlockedDecrement64_nf:
341 return MSVCIntrin::_InterlockedDecrement_nf;
342 case clang::ARM::BI_ReadWriteBarrier:
343 case clang::ARM::BI_ReadBarrier:
344 case clang::ARM::BI_WriteBarrier:
345 return MSVCIntrin::_ReadWriteBarrier;
346 }
347 llvm_unreachable("must return from switch");
348}
349
350// Emit an intrinsic where all operands are of the same type as the result.
351// Depending on mode, this may be a constrained floating-point intrinsic.
352static Value *emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
353 unsigned IntrinsicID,
354 unsigned ConstrainedIntrinsicID,
355 llvm::Type *Ty,
356 ArrayRef<Value *> Args) {
357 Function *F;
358 if (CGF.Builder.getIsFPConstrained())
359 F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Ty);
360 else
361 F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Ty);
362
363 if (CGF.Builder.getIsFPConstrained())
364 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args);
365
366 return CGF.Builder.CreateCall(Callee: F, Args);
367}
368
369static llvm::FixedVectorType *GetNeonType(CodeGenFunction *CGF,
370 NeonTypeFlags TypeFlags,
371 bool HasFastHalfType = true,
372 bool V1Ty = false,
373 bool AllowBFloatArgsAndRet = true) {
374 int IsQuad = TypeFlags.isQuad();
375 switch (TypeFlags.getEltType()) {
376 case NeonTypeFlags::Int8:
377 case NeonTypeFlags::Poly8:
378 case NeonTypeFlags::MFloat8:
379 return llvm::FixedVectorType::get(ElementType: CGF->Int8Ty, NumElts: V1Ty ? 1 : (8 << IsQuad));
380 case NeonTypeFlags::Int16:
381 case NeonTypeFlags::Poly16:
382 return llvm::FixedVectorType::get(ElementType: CGF->Int16Ty, NumElts: V1Ty ? 1 : (4 << IsQuad));
383 case NeonTypeFlags::BFloat16:
384 if (AllowBFloatArgsAndRet)
385 return llvm::FixedVectorType::get(ElementType: CGF->BFloatTy, NumElts: V1Ty ? 1 : (4 << IsQuad));
386 return llvm::FixedVectorType::get(ElementType: CGF->Int16Ty, NumElts: V1Ty ? 1 : (4 << IsQuad));
387 case NeonTypeFlags::Float16:
388 if (HasFastHalfType)
389 return llvm::FixedVectorType::get(ElementType: CGF->HalfTy, NumElts: V1Ty ? 1 : (4 << IsQuad));
390 return llvm::FixedVectorType::get(ElementType: CGF->Int16Ty, NumElts: V1Ty ? 1 : (4 << IsQuad));
391 case NeonTypeFlags::Int32:
392 return llvm::FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: V1Ty ? 1 : (2 << IsQuad));
393 case NeonTypeFlags::Int64:
394 case NeonTypeFlags::Poly64:
395 return llvm::FixedVectorType::get(ElementType: CGF->Int64Ty, NumElts: V1Ty ? 1 : (1 << IsQuad));
396 case NeonTypeFlags::Poly128:
397 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
398 // There is a lot of i128 and f128 API missing.
399 // so we use v16i8 to represent poly128 and get pattern matched.
400 return llvm::FixedVectorType::get(ElementType: CGF->Int8Ty, NumElts: 16);
401 case NeonTypeFlags::Float32:
402 return llvm::FixedVectorType::get(ElementType: CGF->FloatTy, NumElts: V1Ty ? 1 : (2 << IsQuad));
403 case NeonTypeFlags::Float64:
404 return llvm::FixedVectorType::get(ElementType: CGF->DoubleTy, NumElts: V1Ty ? 1 : (1 << IsQuad));
405 }
406 llvm_unreachable("Unknown vector element type!");
407}
408
409static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
410 NeonTypeFlags IntTypeFlags) {
411 int IsQuad = IntTypeFlags.isQuad();
412 switch (IntTypeFlags.getEltType()) {
413 case NeonTypeFlags::Int16:
414 return llvm::FixedVectorType::get(ElementType: CGF->HalfTy, NumElts: (4 << IsQuad));
415 case NeonTypeFlags::Int32:
416 return llvm::FixedVectorType::get(ElementType: CGF->FloatTy, NumElts: (2 << IsQuad));
417 case NeonTypeFlags::Int64:
418 return llvm::FixedVectorType::get(ElementType: CGF->DoubleTy, NumElts: (1 << IsQuad));
419 default:
420 llvm_unreachable("Type can't be converted to floating-point!");
421 }
422}
423
424Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C,
425 const ElementCount &Count) {
426 Value *SV = llvm::ConstantVector::getSplat(EC: Count, Elt: C);
427 return Builder.CreateShuffleVector(V1: V, V2: V, Mask: SV, Name: "lane");
428}
429
430Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
431 ElementCount EC = cast<llvm::VectorType>(Val: V->getType())->getElementCount();
432 return EmitNeonSplat(V, C, Count: EC);
433}
434
435Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
436 const char *name,
437 unsigned shift, bool rightshift) {
438 unsigned j = 0;
439 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
440 ai != ae; ++ai, ++j) {
441 if (F->isConstrainedFPIntrinsic())
442 if (ai->getType()->isMetadataTy())
443 continue;
444 if (shift > 0 && shift == j)
445 Ops[j] = EmitNeonShiftVector(V: Ops[j], Ty: ai->getType(), negateForRightShift: rightshift);
446 else
447 Ops[j] = Builder.CreateBitCast(V: Ops[j], DestTy: ai->getType(), Name: name);
448 }
449
450 if (F->isConstrainedFPIntrinsic())
451 return Builder.CreateConstrainedFPCall(Callee: F, Args: Ops, Name: name);
452 return Builder.CreateCall(Callee: F, Args: Ops, Name: name);
453}
454
455Value *CodeGenFunction::EmitFP8NeonCall(unsigned IID,
456 ArrayRef<llvm::Type *> Tys,
457 SmallVectorImpl<Value *> &Ops,
458 const CallExpr *E, const char *name) {
459 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_set_fpmr),
460 Args: Ops.pop_back_val());
461 return EmitNeonCall(F: CGM.getIntrinsic(IID, Tys), Ops, name);
462}
463
464llvm::Value *CodeGenFunction::EmitFP8NeonFDOTCall(
465 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
466 SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
467
468 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
469 RetTy->getPrimitiveSizeInBits();
470 llvm::Type *Tys[] = {llvm::FixedVectorType::get(ElementType: RetTy, NumElts: ElemCount),
471 Ops[1]->getType()};
472 if (ExtendLaneArg) {
473 auto *VT = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16);
474 Ops[2] = Builder.CreateInsertVector(DstType: VT, SrcVec: PoisonValue::get(T: VT), SubVec: Ops[2],
475 Idx: uint64_t(0));
476 }
477 return EmitFP8NeonCall(IID, Tys, Ops, E, name);
478}
479
480llvm::Value *CodeGenFunction::EmitFP8NeonFMLACall(
481 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
482 SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
483
484 if (ExtendLaneArg) {
485 auto *VT = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16);
486 Ops[2] = Builder.CreateInsertVector(DstType: VT, SrcVec: PoisonValue::get(T: VT), SubVec: Ops[2],
487 Idx: uint64_t(0));
488 }
489 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
490 RetTy->getPrimitiveSizeInBits();
491 return EmitFP8NeonCall(IID, Tys: {llvm::FixedVectorType::get(ElementType: RetTy, NumElts: ElemCount)},
492 Ops, E, name);
493}
494
495Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
496 bool neg) {
497 int SV = cast<ConstantInt>(Val: V)->getSExtValue();
498 return ConstantInt::getSigned(Ty, V: neg ? -SV : SV);
499}
500
501Value *CodeGenFunction::EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
502 llvm::Type *Ty1, bool Extract,
503 SmallVectorImpl<llvm::Value *> &Ops,
504 const CallExpr *E,
505 const char *name) {
506 llvm::Type *Tys[] = {Ty0, Ty1};
507 if (Extract) {
508 // Op[0] is mfloat8x16_t, but the intrinsic converts only the lower part of
509 // the vector.
510 Tys[1] = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 8);
511 Ops[0] = Builder.CreateExtractVector(DstType: Tys[1], SrcVec: Ops[0], Idx: uint64_t(0));
512 }
513 return EmitFP8NeonCall(IID, Tys, Ops, E, name);
514}
515
516// Right-shift a vector by a constant.
517Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
518 llvm::Type *Ty, bool usgn,
519 const char *name) {
520 llvm::VectorType *VTy = cast<llvm::VectorType>(Val: Ty);
521
522 int ShiftAmt = cast<ConstantInt>(Val: Shift)->getSExtValue();
523 int EltSize = VTy->getScalarSizeInBits();
524
525 Vec = Builder.CreateBitCast(V: Vec, DestTy: Ty);
526
527 // lshr/ashr are undefined when the shift amount is equal to the vector
528 // element size.
529 if (ShiftAmt == EltSize) {
530 if (usgn) {
531 // Right-shifting an unsigned value by its size yields 0.
532 return llvm::ConstantAggregateZero::get(Ty: VTy);
533 } else {
534 // Right-shifting a signed value by its size is equivalent
535 // to a shift of size-1.
536 --ShiftAmt;
537 Shift = ConstantInt::get(Ty: VTy->getElementType(), V: ShiftAmt);
538 }
539 }
540
541 Shift = EmitNeonShiftVector(V: Shift, Ty, neg: false);
542 if (usgn)
543 return Builder.CreateLShr(LHS: Vec, RHS: Shift, Name: name);
544 return Builder.CreateAShr(LHS: Vec, RHS: Shift, Name: name);
545}
546
547// clang-format off
548static const ARMNeonVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
549 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
550 NEONMAP0(splat_lane_v),
551 NEONMAP0(splat_laneq_v),
552 NEONMAP0(splatq_lane_v),
553 NEONMAP0(splatq_laneq_v),
554 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
555 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
556 NEONMAP1(vabs_v, arm_neon_vabs, 0),
557 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
558 NEONMAP0(vadd_v),
559 NEONMAP0(vaddhn_v),
560 NEONMAP0(vaddq_v),
561 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
562 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
563 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
564 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
565 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
566 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
567 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
568 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
569 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
570 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
571 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
572 NEONMAP1(vcadd_rot270_f16, arm_neon_vcadd_rot270, Add1ArgType),
573 NEONMAP1(vcadd_rot270_f32, arm_neon_vcadd_rot270, Add1ArgType),
574 NEONMAP1(vcadd_rot90_f16, arm_neon_vcadd_rot90, Add1ArgType),
575 NEONMAP1(vcadd_rot90_f32, arm_neon_vcadd_rot90, Add1ArgType),
576 NEONMAP1(vcaddq_rot270_f16, arm_neon_vcadd_rot270, Add1ArgType),
577 NEONMAP1(vcaddq_rot270_f32, arm_neon_vcadd_rot270, Add1ArgType),
578 NEONMAP1(vcaddq_rot270_f64, arm_neon_vcadd_rot270, Add1ArgType),
579 NEONMAP1(vcaddq_rot90_f16, arm_neon_vcadd_rot90, Add1ArgType),
580 NEONMAP1(vcaddq_rot90_f32, arm_neon_vcadd_rot90, Add1ArgType),
581 NEONMAP1(vcaddq_rot90_f64, arm_neon_vcadd_rot90, Add1ArgType),
582 NEONMAP1(vcage_v, arm_neon_vacge, 0),
583 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
584 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
585 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
586 NEONMAP1(vcale_v, arm_neon_vacge, 0),
587 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
588 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
589 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
590 NEONMAP0(vceqz_v),
591 NEONMAP0(vceqzq_v),
592 NEONMAP0(vcgez_v),
593 NEONMAP0(vcgezq_v),
594 NEONMAP0(vcgtz_v),
595 NEONMAP0(vcgtzq_v),
596 NEONMAP0(vclez_v),
597 NEONMAP0(vclezq_v),
598 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
599 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
600 NEONMAP0(vcltz_v),
601 NEONMAP0(vcltzq_v),
602 NEONMAP1(vclz_v, ctlz, Add1ArgType),
603 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
604 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
605 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
606 NEONMAP0(vcvt_f16_s16),
607 NEONMAP0(vcvt_f16_u16),
608 NEONMAP0(vcvt_f32_v),
609 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
610 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
611 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
612 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
613 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
614 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
615 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
616 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
617 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
618 NEONMAP0(vcvt_s16_f16),
619 NEONMAP0(vcvt_s32_v),
620 NEONMAP0(vcvt_s64_v),
621 NEONMAP0(vcvt_u16_f16),
622 NEONMAP0(vcvt_u32_v),
623 NEONMAP0(vcvt_u64_v),
624 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
625 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
626 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
627 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
628 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
629 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
630 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
631 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
632 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
633 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
634 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
635 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
636 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
637 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
638 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
639 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
640 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
641 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
642 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
643 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
644 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
645 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
646 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
647 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
648 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
649 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
650 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
651 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
652 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
653 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
654 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
655 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
656 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
657 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
658 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
659 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
660 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
661 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
662 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
663 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
664 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
665 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
666 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
667 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
668 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
669 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
670 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
671 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
672 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
673 NEONMAP0(vcvtq_f16_s16),
674 NEONMAP0(vcvtq_f16_u16),
675 NEONMAP0(vcvtq_f32_v),
676 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
677 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
678 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
679 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
680 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
681 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
682 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
683 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
684 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
685 NEONMAP0(vcvtq_s16_f16),
686 NEONMAP0(vcvtq_s32_v),
687 NEONMAP0(vcvtq_s64_v),
688 NEONMAP0(vcvtq_u16_f16),
689 NEONMAP0(vcvtq_u32_v),
690 NEONMAP0(vcvtq_u64_v),
691 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
692 NEONMAP1(vdot_u32, arm_neon_udot, 0),
693 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
694 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
695 NEONMAP0(vext_v),
696 NEONMAP0(vextq_v),
697 NEONMAP0(vfma_v),
698 NEONMAP0(vfmaq_v),
699 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
700 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
701 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
702 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
703 NEONMAP0(vld1_dup_v),
704 NEONMAP1(vld1_v, arm_neon_vld1, 0),
705 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
706 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
707 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
708 NEONMAP0(vld1q_dup_v),
709 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
710 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
711 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
712 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
713 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
714 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
715 NEONMAP1(vld2_v, arm_neon_vld2, 0),
716 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
717 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
718 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
719 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
720 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
721 NEONMAP1(vld3_v, arm_neon_vld3, 0),
722 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
723 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
724 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
725 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
726 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
727 NEONMAP1(vld4_v, arm_neon_vld4, 0),
728 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
729 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
730 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
731 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
732 NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
733 NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
734 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
735 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
736 NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
737 NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
738 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
739 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
740 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
741 NEONMAP0(vmovl_v),
742 NEONMAP0(vmovn_v),
743 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
744 NEONMAP0(vmull_v),
745 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
746 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
747 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
748 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
749 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
750 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
751 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
752 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
753 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
754 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
755 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
756 NEONMAP2(vqadd_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
757 NEONMAP2(vqaddq_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
758 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
759 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
760 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
761 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
762 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
763 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
764 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
765 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
766 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
767 NEONMAP1(vqrdmlah_s16, arm_neon_vqrdmlah, Add1ArgType),
768 NEONMAP1(vqrdmlah_s32, arm_neon_vqrdmlah, Add1ArgType),
769 NEONMAP1(vqrdmlahq_s16, arm_neon_vqrdmlah, Add1ArgType),
770 NEONMAP1(vqrdmlahq_s32, arm_neon_vqrdmlah, Add1ArgType),
771 NEONMAP1(vqrdmlsh_s16, arm_neon_vqrdmlsh, Add1ArgType),
772 NEONMAP1(vqrdmlsh_s32, arm_neon_vqrdmlsh, Add1ArgType),
773 NEONMAP1(vqrdmlshq_s16, arm_neon_vqrdmlsh, Add1ArgType),
774 NEONMAP1(vqrdmlshq_s32, arm_neon_vqrdmlsh, Add1ArgType),
775 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
776 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
777 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
778 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
779 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
780 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
781 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
782 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
783 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
784 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
785 NEONMAP2(vqsub_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
786 NEONMAP2(vqsubq_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
787 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
788 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
789 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
790 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
791 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
792 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
793 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
794 NEONMAP1(vrnd_v, trunc, Add1ArgType),
795 NEONMAP1(vrnda_v, round, Add1ArgType),
796 NEONMAP1(vrndaq_v, round, Add1ArgType),
797 NEONMAP0(vrndi_v),
798 NEONMAP0(vrndiq_v),
799 NEONMAP1(vrndm_v, floor, Add1ArgType),
800 NEONMAP1(vrndmq_v, floor, Add1ArgType),
801 NEONMAP1(vrndn_v, roundeven, Add1ArgType),
802 NEONMAP1(vrndnq_v, roundeven, Add1ArgType),
803 NEONMAP1(vrndp_v, ceil, Add1ArgType),
804 NEONMAP1(vrndpq_v, ceil, Add1ArgType),
805 NEONMAP1(vrndq_v, trunc, Add1ArgType),
806 NEONMAP1(vrndx_v, rint, Add1ArgType),
807 NEONMAP1(vrndxq_v, rint, Add1ArgType),
808 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
809 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
810 NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
811 NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
812 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
813 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
814 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
815 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
816 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
817 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
818 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
819 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
820 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
821 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
822 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
823 NEONMAP0(vshl_n_v),
824 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
825 NEONMAP0(vshll_n_v),
826 NEONMAP0(vshlq_n_v),
827 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
828 NEONMAP0(vshr_n_v),
829 NEONMAP0(vshrn_n_v),
830 NEONMAP0(vshrq_n_v),
831 NEONMAP1(vst1_v, arm_neon_vst1, 0),
832 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
833 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
834 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
835 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
836 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
837 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
838 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
839 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
840 NEONMAP1(vst2_v, arm_neon_vst2, 0),
841 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
842 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
843 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
844 NEONMAP1(vst3_v, arm_neon_vst3, 0),
845 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
846 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
847 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
848 NEONMAP1(vst4_v, arm_neon_vst4, 0),
849 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
850 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
851 NEONMAP0(vsubhn_v),
852 NEONMAP0(vtrn_v),
853 NEONMAP0(vtrnq_v),
854 NEONMAP0(vtst_v),
855 NEONMAP0(vtstq_v),
856 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
857 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
858 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
859 NEONMAP0(vuzp_v),
860 NEONMAP0(vuzpq_v),
861 NEONMAP0(vzip_v),
862 NEONMAP0(vzipq_v)
863};
864
865// clang-format on
866
867// Some intrinsics are equivalent for codegen.
868static const std::pair<unsigned, unsigned> NEONEquivalentIntrinsicMap[] = {
869 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
870 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
871 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
872 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
873 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
874 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
875 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
876 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
877 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
878 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
879 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
880 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
881 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
882 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
883 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
884 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
885 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
886 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
887 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
888 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
889 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
890 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
891 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
892 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
893 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
894 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
895 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
896 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
897 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
898 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
899 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
900 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
901 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
902 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
903 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
904 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
905 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
906 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
907 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
908 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
909 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
910 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
911 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
912 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
913 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
914 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
915 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
916 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
917 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
918 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
919 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
920 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
921 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
922 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
923 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
924 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
925 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
926 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
927 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
928 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
929 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
930 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
931 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
932 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
933 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
934 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
935 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
936 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
937 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
938 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
939 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
940 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
941 // The mangling rules cause us to have one ID for each type for vldap1(q)_lane
942 // and vstl1(q)_lane, but codegen is equivalent for all of them. Choose an
943 // arbitrary one to be handled as tha canonical variation.
944 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
945 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
946 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
947 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
948 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
949 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
950 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
951 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
952 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
953 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
954 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
955 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
956};
957
958#undef NEONMAP0
959#undef NEONMAP1
960#undef NEONMAP2
961
962#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
963 {SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
964
965#define SVEMAP2(NameBase, TypeModifier) \
966 {SVE::BI__builtin_sve_##NameBase, 0, TypeModifier}
967static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SVEIntrinsicMap[] = {
968#define GET_SVE_LLVM_INTRINSIC_MAP
969#include "clang/Basic/arm_sve_builtin_cg.inc"
970#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
971#undef GET_SVE_LLVM_INTRINSIC_MAP
972};
973
974#undef SVEMAP1
975#undef SVEMAP2
976
977#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
978 {SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
979
980#define SMEMAP2(NameBase, TypeModifier) \
981 {SME::BI__builtin_sme_##NameBase, 0, TypeModifier}
982static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SMEIntrinsicMap[] = {
983#define GET_SME_LLVM_INTRINSIC_MAP
984#include "clang/Basic/arm_sme_builtin_cg.inc"
985#undef GET_SME_LLVM_INTRINSIC_MAP
986};
987
988#undef SMEMAP1
989#undef SMEMAP2
990
991static bool NEONSIMDIntrinsicsProvenSorted = false;
992
993static bool AArch64SIMDIntrinsicsProvenSorted = false;
994static bool AArch64SISDIntrinsicsProvenSorted = false;
995static bool AArch64SVEIntrinsicsProvenSorted = false;
996static bool AArch64SMEIntrinsicsProvenSorted = false;
997
998// Check if Builtin `BuiltinId` is present in `IntrinsicMap`. If yes, returns
999// the corresponding info struct.
1000template <typename IntrinsicInfo>
1001static const IntrinsicInfo *
1002findARMVectorIntrinsicInMap(ArrayRef<IntrinsicInfo> IntrinsicMap,
1003 unsigned BuiltinID, bool &MapProvenSorted) {
1004
1005#ifndef NDEBUG
1006 if (!MapProvenSorted) {
1007 assert(llvm::is_sorted(IntrinsicMap));
1008 MapProvenSorted = true;
1009 }
1010#endif
1011
1012 const IntrinsicInfo *Builtin = llvm::lower_bound(IntrinsicMap, BuiltinID);
1013
1014 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
1015 return Builtin;
1016
1017 return nullptr;
1018}
1019
1020Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
1021 unsigned Modifier,
1022 llvm::Type *ArgType,
1023 const CallExpr *E) {
1024 int VectorSize = 0;
1025 if (Modifier & Use64BitVectors)
1026 VectorSize = 64;
1027 else if (Modifier & Use128BitVectors)
1028 VectorSize = 128;
1029
1030 // Return type.
1031 SmallVector<llvm::Type *, 3> Tys;
1032 if (Modifier & AddRetType) {
1033 llvm::Type *Ty = ConvertType(T: E->getCallReturnType(Ctx: getContext()));
1034 if (Modifier & VectorizeRetType)
1035 Ty = llvm::FixedVectorType::get(
1036 ElementType: Ty, NumElts: VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
1037
1038 Tys.push_back(Elt: Ty);
1039 }
1040
1041 // Arguments.
1042 if (Modifier & VectorizeArgTypes) {
1043 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
1044 ArgType = llvm::FixedVectorType::get(ElementType: ArgType, NumElts: Elts);
1045 }
1046
1047 if (Modifier & (Add1ArgType | Add2ArgTypes))
1048 Tys.push_back(Elt: ArgType);
1049
1050 if (Modifier & Add2ArgTypes)
1051 Tys.push_back(Elt: ArgType);
1052
1053 if (Modifier & InventFloatType)
1054 Tys.push_back(Elt: FloatTy);
1055
1056 return CGM.getIntrinsic(IID: IntrinsicID, Tys);
1057}
1058
1059//===----------------------------------------------------------------------===//
1060// Emit-helpers
1061//===----------------------------------------------------------------------===//
1062static Value *EmitCommonNeonSISDBuiltinExpr(
1063 CodeGenFunction &CGF, const ARMNeonVectorIntrinsicInfo &SISDInfo,
1064 SmallVectorImpl<Value *> &Ops, const CallExpr *E) {
1065 assert(SISDInfo.LLVMIntrinsic && "Generic code assumes a valid intrinsic");
1066
1067 switch (SISDInfo.BuiltinID) {
1068 case NEON::BI__builtin_neon_vcled_s64:
1069 case NEON::BI__builtin_neon_vcled_u64:
1070 case NEON::BI__builtin_neon_vcles_f32:
1071 case NEON::BI__builtin_neon_vcled_f64:
1072 case NEON::BI__builtin_neon_vcltd_s64:
1073 case NEON::BI__builtin_neon_vcltd_u64:
1074 case NEON::BI__builtin_neon_vclts_f32:
1075 case NEON::BI__builtin_neon_vcltd_f64:
1076 case NEON::BI__builtin_neon_vcales_f32:
1077 case NEON::BI__builtin_neon_vcaled_f64:
1078 case NEON::BI__builtin_neon_vcalts_f32:
1079 case NEON::BI__builtin_neon_vcaltd_f64:
1080 // Only one direction of comparisons actually exist, cmle is actually a cmge
1081 // with swapped operands. The table gives us the right intrinsic but we
1082 // still need to do the swap.
1083 std::swap(a&: Ops[0], b&: Ops[1]);
1084 break;
1085 }
1086
1087 // Use fptosi.sat/fptoui.sat unless under strict FP.
1088 unsigned LLVMIntrinsic = SISDInfo.LLVMIntrinsic;
1089 if (!CGF.Builder.getIsFPConstrained()) {
1090 if (LLVMIntrinsic == Intrinsic::aarch64_neon_fcvtzs)
1091 LLVMIntrinsic = Intrinsic::fptosi_sat;
1092 else if (LLVMIntrinsic == Intrinsic::aarch64_neon_fcvtzu)
1093 LLVMIntrinsic = Intrinsic::fptoui_sat;
1094 }
1095 llvm::Type *ArgTy = CGF.ConvertType(T: E->getArg(Arg: 0)->getType());
1096 Function *F = CGF.LookupNeonLLVMIntrinsic(IntrinsicID: LLVMIntrinsic,
1097 Modifier: SISDInfo.TypeModifier, ArgType: ArgTy, E);
1098
1099 int j = 0;
1100 ConstantInt *C0 = ConstantInt::get(Ty: CGF.SizeTy, V: 0);
1101 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
1102 ai != ae; ++ai, ++j) {
1103 llvm::Type *ArgTy = ai->getType();
1104 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
1105 ArgTy->getPrimitiveSizeInBits())
1106 continue;
1107 assert(
1108 ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy() &&
1109 "Expecting vector LLVM intrinsic type and scalar Clang builtin type!");
1110
1111 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
1112 // it before inserting.
1113 Ops[j] = CGF.Builder.CreateTruncOrBitCast(
1114 V: Ops[j], DestTy: cast<llvm::VectorType>(Val: ArgTy)->getElementType());
1115 Ops[j] =
1116 CGF.Builder.CreateInsertElement(Vec: PoisonValue::get(T: ArgTy), NewElt: Ops[j], Idx: C0);
1117 }
1118
1119 Value *Result = CGF.EmitNeonCall(F, Ops, name: SISDInfo.NameHint);
1120 llvm::Type *ResultType = CGF.ConvertType(T: E->getType());
1121 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
1122 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
1123 return CGF.Builder.CreateExtractElement(Vec: Result, Idx: C0);
1124
1125 return CGF.Builder.CreateBitCast(V: Result, DestTy: ResultType, Name: SISDInfo.NameHint);
1126}
1127
1128Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
1129 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
1130 const char *NameHint, unsigned Modifier, const CallExpr *E,
1131 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1,
1132 llvm::Triple::ArchType Arch) {
1133
1134 // Extract the trailing immediate argument that encodes the type discriminator
1135 // for this overloaded intrinsic.
1136 // TODO: Move to the parent code that takes care of argument processing.
1137 const Expr *Arg = E->getArg(Arg: E->getNumArgs() - 1);
1138 std::optional<llvm::APSInt> NeonTypeConst =
1139 Arg->getIntegerConstantExpr(Ctx: getContext());
1140 if (!NeonTypeConst)
1141 return nullptr;
1142
1143 // Determine the type of this overloaded NEON intrinsic.
1144 NeonTypeFlags Type(NeonTypeConst->getZExtValue());
1145 const bool Usgn = Type.isUnsigned();
1146 const bool Quad = Type.isQuad();
1147 const bool Floating = Type.isFloatingPoint();
1148 const bool HasFastHalfType = getTarget().hasFastHalfType();
1149 const bool AllowBFloatArgsAndRet =
1150 getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
1151
1152 llvm::FixedVectorType *VTy =
1153 GetNeonType(CGF: this, TypeFlags: Type, HasFastHalfType, V1Ty: false, AllowBFloatArgsAndRet);
1154 llvm::Type *Ty = VTy;
1155 if (!Ty)
1156 return nullptr;
1157
1158 auto getAlignmentValue32 = [&](Address addr) -> Value* {
1159 return Builder.getInt32(C: addr.getAlignment().getQuantity());
1160 };
1161
1162 unsigned Int = LLVMIntrinsic;
1163 if ((Modifier & UnsignedAlts) && !Usgn)
1164 Int = AltLLVMIntrinsic;
1165
1166 switch (BuiltinID) {
1167 default: break;
1168 case NEON::BI__builtin_neon_splat_lane_v:
1169 case NEON::BI__builtin_neon_splat_laneq_v:
1170 case NEON::BI__builtin_neon_splatq_lane_v:
1171 case NEON::BI__builtin_neon_splatq_laneq_v: {
1172 auto NumElements = VTy->getElementCount();
1173 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
1174 NumElements = NumElements * 2;
1175 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
1176 NumElements = NumElements.divideCoefficientBy(RHS: 2);
1177
1178 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: VTy);
1179 return EmitNeonSplat(V: Ops[0], C: cast<ConstantInt>(Val: Ops[1]), Count: NumElements);
1180 }
1181 case NEON::BI__builtin_neon_vpadd_v:
1182 case NEON::BI__builtin_neon_vpaddq_v:
1183 // We don't allow fp/int overloading of intrinsics.
1184 if (VTy->getElementType()->isFloatingPointTy() &&
1185 Int == Intrinsic::aarch64_neon_addp)
1186 Int = Intrinsic::aarch64_neon_faddp;
1187 break;
1188 case NEON::BI__builtin_neon_vabs_v:
1189 case NEON::BI__builtin_neon_vabsq_v:
1190 if (VTy->getElementType()->isFloatingPointTy())
1191 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::fabs, Tys: Ty), Ops, name: "vabs");
1192 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys: Ty), Ops, name: "vabs");
1193 case NEON::BI__builtin_neon_vadd_v:
1194 case NEON::BI__builtin_neon_vaddq_v: {
1195 llvm::Type *VTy = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Quad ? 16 : 8);
1196 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: VTy);
1197 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: VTy);
1198 Ops[0] = Builder.CreateXor(LHS: Ops[0], RHS: Ops[1]);
1199 return Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1200 }
1201 case NEON::BI__builtin_neon_vaddhn_v: {
1202 llvm::FixedVectorType *SrcTy =
1203 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1204
1205 // %sum = add <4 x i32> %lhs, %rhs
1206 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1207 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: SrcTy);
1208 Ops[0] = Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1], Name: "vaddhn");
1209
1210 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
1211 Constant *ShiftAmt =
1212 ConstantInt::get(Ty: SrcTy, V: SrcTy->getScalarSizeInBits() / 2);
1213 Ops[0] = Builder.CreateLShr(LHS: Ops[0], RHS: ShiftAmt, Name: "vaddhn");
1214
1215 // %res = trunc <4 x i32> %high to <4 x i16>
1216 return Builder.CreateTrunc(V: Ops[0], DestTy: VTy, Name: "vaddhn");
1217 }
1218 case NEON::BI__builtin_neon_vcale_v:
1219 case NEON::BI__builtin_neon_vcaleq_v:
1220 case NEON::BI__builtin_neon_vcalt_v:
1221 case NEON::BI__builtin_neon_vcaltq_v:
1222 std::swap(a&: Ops[0], b&: Ops[1]);
1223 [[fallthrough]];
1224 case NEON::BI__builtin_neon_vcage_v:
1225 case NEON::BI__builtin_neon_vcageq_v:
1226 case NEON::BI__builtin_neon_vcagt_v:
1227 case NEON::BI__builtin_neon_vcagtq_v: {
1228 llvm::Type *Ty;
1229 switch (VTy->getScalarSizeInBits()) {
1230 default: llvm_unreachable("unexpected type");
1231 case 32:
1232 Ty = FloatTy;
1233 break;
1234 case 64:
1235 Ty = DoubleTy;
1236 break;
1237 case 16:
1238 Ty = HalfTy;
1239 break;
1240 }
1241 auto *VecFlt = llvm::FixedVectorType::get(ElementType: Ty, NumElts: VTy->getNumElements());
1242 llvm::Type *Tys[] = { VTy, VecFlt };
1243 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1244 return EmitNeonCall(F, Ops, name: NameHint);
1245 }
1246 case NEON::BI__builtin_neon_vceqz_v:
1247 case NEON::BI__builtin_neon_vceqzq_v:
1248 return EmitAArch64CompareBuiltinExpr(
1249 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OEQ : ICmpInst::ICMP_EQ, Name: "vceqz");
1250 case NEON::BI__builtin_neon_vcgez_v:
1251 case NEON::BI__builtin_neon_vcgezq_v:
1252 return EmitAArch64CompareBuiltinExpr(
1253 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OGE : ICmpInst::ICMP_SGE,
1254 Name: "vcgez");
1255 case NEON::BI__builtin_neon_vclez_v:
1256 case NEON::BI__builtin_neon_vclezq_v:
1257 return EmitAArch64CompareBuiltinExpr(
1258 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OLE : ICmpInst::ICMP_SLE,
1259 Name: "vclez");
1260 case NEON::BI__builtin_neon_vcgtz_v:
1261 case NEON::BI__builtin_neon_vcgtzq_v:
1262 return EmitAArch64CompareBuiltinExpr(
1263 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OGT : ICmpInst::ICMP_SGT,
1264 Name: "vcgtz");
1265 case NEON::BI__builtin_neon_vcltz_v:
1266 case NEON::BI__builtin_neon_vcltzq_v:
1267 return EmitAArch64CompareBuiltinExpr(
1268 Op: Ops[0], Ty, Pred: Floating ? ICmpInst::FCMP_OLT : ICmpInst::ICMP_SLT,
1269 Name: "vcltz");
1270 case NEON::BI__builtin_neon_vclz_v:
1271 case NEON::BI__builtin_neon_vclzq_v:
1272 // We generate target-independent intrinsic, which needs a second argument
1273 // for whether or not clz of zero is undefined; on ARM it isn't.
1274 Ops.push_back(Elt: Builder.getInt1(V: getTarget().isCLZForZeroUndef()));
1275 break;
1276 case NEON::BI__builtin_neon_vcvt_f32_v:
1277 case NEON::BI__builtin_neon_vcvtq_f32_v:
1278 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1279 Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float32, false, Quad),
1280 HasFastHalfType);
1281 return Usgn ? Builder.CreateUIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt")
1282 : Builder.CreateSIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt");
1283 case NEON::BI__builtin_neon_vcvt_f16_s16:
1284 case NEON::BI__builtin_neon_vcvt_f16_u16:
1285 case NEON::BI__builtin_neon_vcvtq_f16_s16:
1286 case NEON::BI__builtin_neon_vcvtq_f16_u16:
1287 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1288 Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float16, false, Quad),
1289 HasFastHalfType);
1290 return Usgn ? Builder.CreateUIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt")
1291 : Builder.CreateSIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt");
1292 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
1293 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
1294 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
1295 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
1296 llvm::Type *Tys[2] = { GetFloatNeonType(CGF: this, IntTypeFlags: Type), Ty };
1297 Function *F = CGM.getIntrinsic(IID: Int, Tys);
1298 return EmitNeonCall(F, Ops, name: "vcvt_n");
1299 }
1300 case NEON::BI__builtin_neon_vcvt_n_f32_v:
1301 case NEON::BI__builtin_neon_vcvt_n_f64_v:
1302 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
1303 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
1304 llvm::Type *Tys[2] = { GetFloatNeonType(CGF: this, IntTypeFlags: Type), Ty };
1305 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
1306 Function *F = CGM.getIntrinsic(IID: Int, Tys);
1307 return EmitNeonCall(F, Ops, name: "vcvt_n");
1308 }
1309 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
1310 case NEON::BI__builtin_neon_vcvt_n_s32_v:
1311 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
1312 case NEON::BI__builtin_neon_vcvt_n_u32_v:
1313 case NEON::BI__builtin_neon_vcvt_n_s64_v:
1314 case NEON::BI__builtin_neon_vcvt_n_u64_v:
1315 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
1316 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
1317 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
1318 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
1319 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
1320 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
1321 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
1322 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1323 return EmitNeonCall(F, Ops, name: "vcvt_n");
1324 }
1325 case NEON::BI__builtin_neon_vcvt_s32_v:
1326 case NEON::BI__builtin_neon_vcvt_u32_v:
1327 case NEON::BI__builtin_neon_vcvt_s64_v:
1328 case NEON::BI__builtin_neon_vcvt_u64_v:
1329 case NEON::BI__builtin_neon_vcvt_s16_f16:
1330 case NEON::BI__builtin_neon_vcvt_u16_f16:
1331 case NEON::BI__builtin_neon_vcvtq_s32_v:
1332 case NEON::BI__builtin_neon_vcvtq_u32_v:
1333 case NEON::BI__builtin_neon_vcvtq_s64_v:
1334 case NEON::BI__builtin_neon_vcvtq_u64_v:
1335 case NEON::BI__builtin_neon_vcvtq_s16_f16:
1336 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
1337 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: GetFloatNeonType(CGF: this, IntTypeFlags: Type));
1338 if (Int) {
1339 // AArch64: use fptosi.sat/fptoui.sat unless under strict FP.
1340 if (!Builder.getIsFPConstrained())
1341 Int = Usgn ? Intrinsic::fptoui_sat : Intrinsic::fptosi_sat;
1342 llvm::Type *Tys[2] = {Ty, Ops[0]->getType()};
1343 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtz");
1344 }
1345 // FIXME: ARM uses plain fptoui/fptosi which have UB on out-of-range
1346 // values. These should also use saturating intrinsics.
1347 return Usgn ? Builder.CreateFPToUI(V: Ops[0], DestTy: Ty, Name: "vcvt")
1348 : Builder.CreateFPToSI(V: Ops[0], DestTy: Ty, Name: "vcvt");
1349 }
1350 case NEON::BI__builtin_neon_vcvta_s16_f16:
1351 case NEON::BI__builtin_neon_vcvta_s32_v:
1352 case NEON::BI__builtin_neon_vcvta_s64_v:
1353 case NEON::BI__builtin_neon_vcvta_u16_f16:
1354 case NEON::BI__builtin_neon_vcvta_u32_v:
1355 case NEON::BI__builtin_neon_vcvta_u64_v:
1356 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
1357 case NEON::BI__builtin_neon_vcvtaq_s32_v:
1358 case NEON::BI__builtin_neon_vcvtaq_s64_v:
1359 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
1360 case NEON::BI__builtin_neon_vcvtaq_u32_v:
1361 case NEON::BI__builtin_neon_vcvtaq_u64_v:
1362 case NEON::BI__builtin_neon_vcvtn_s16_f16:
1363 case NEON::BI__builtin_neon_vcvtn_s32_v:
1364 case NEON::BI__builtin_neon_vcvtn_s64_v:
1365 case NEON::BI__builtin_neon_vcvtn_u16_f16:
1366 case NEON::BI__builtin_neon_vcvtn_u32_v:
1367 case NEON::BI__builtin_neon_vcvtn_u64_v:
1368 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
1369 case NEON::BI__builtin_neon_vcvtnq_s32_v:
1370 case NEON::BI__builtin_neon_vcvtnq_s64_v:
1371 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
1372 case NEON::BI__builtin_neon_vcvtnq_u32_v:
1373 case NEON::BI__builtin_neon_vcvtnq_u64_v:
1374 case NEON::BI__builtin_neon_vcvtp_s16_f16:
1375 case NEON::BI__builtin_neon_vcvtp_s32_v:
1376 case NEON::BI__builtin_neon_vcvtp_s64_v:
1377 case NEON::BI__builtin_neon_vcvtp_u16_f16:
1378 case NEON::BI__builtin_neon_vcvtp_u32_v:
1379 case NEON::BI__builtin_neon_vcvtp_u64_v:
1380 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
1381 case NEON::BI__builtin_neon_vcvtpq_s32_v:
1382 case NEON::BI__builtin_neon_vcvtpq_s64_v:
1383 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
1384 case NEON::BI__builtin_neon_vcvtpq_u32_v:
1385 case NEON::BI__builtin_neon_vcvtpq_u64_v:
1386 case NEON::BI__builtin_neon_vcvtm_s16_f16:
1387 case NEON::BI__builtin_neon_vcvtm_s32_v:
1388 case NEON::BI__builtin_neon_vcvtm_s64_v:
1389 case NEON::BI__builtin_neon_vcvtm_u16_f16:
1390 case NEON::BI__builtin_neon_vcvtm_u32_v:
1391 case NEON::BI__builtin_neon_vcvtm_u64_v:
1392 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
1393 case NEON::BI__builtin_neon_vcvtmq_s32_v:
1394 case NEON::BI__builtin_neon_vcvtmq_s64_v:
1395 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
1396 case NEON::BI__builtin_neon_vcvtmq_u32_v:
1397 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
1398 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
1399 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: NameHint);
1400 }
1401 case NEON::BI__builtin_neon_vcvtx_f32_v: {
1402 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
1403 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: NameHint);
1404
1405 }
1406 case NEON::BI__builtin_neon_vext_v:
1407 case NEON::BI__builtin_neon_vextq_v: {
1408 int CV = cast<ConstantInt>(Val: Ops[2])->getSExtValue();
1409 SmallVector<int, 16> Indices;
1410 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1411 Indices.push_back(Elt: i+CV);
1412
1413 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1414 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1415 return Builder.CreateShuffleVector(V1: Ops[0], V2: Ops[1], Mask: Indices, Name: "vext");
1416 }
1417 case NEON::BI__builtin_neon_vfma_v:
1418 case NEON::BI__builtin_neon_vfmaq_v: {
1419 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1420 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1421 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1422
1423 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
1424 return emitCallMaybeConstrainedFPBuiltin(
1425 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
1426 Args: {Ops[1], Ops[2], Ops[0]});
1427 }
1428 case NEON::BI__builtin_neon_vld1_x2_v:
1429 case NEON::BI__builtin_neon_vld1q_x2_v:
1430 case NEON::BI__builtin_neon_vld1_x3_v:
1431 case NEON::BI__builtin_neon_vld1q_x3_v:
1432 case NEON::BI__builtin_neon_vld1_x4_v:
1433 case NEON::BI__builtin_neon_vld1q_x4_v: {
1434 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
1435 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1436 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld1xN");
1437 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
1438 }
1439 case NEON::BI__builtin_neon_vld1_v:
1440 case NEON::BI__builtin_neon_vld1q_v: {
1441 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1442 Ops.push_back(Elt: getAlignmentValue32(PtrOp0));
1443 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "vld1");
1444 }
1445 case NEON::BI__builtin_neon_vld2_v:
1446 case NEON::BI__builtin_neon_vld2q_v:
1447 case NEON::BI__builtin_neon_vld3_v:
1448 case NEON::BI__builtin_neon_vld3q_v:
1449 case NEON::BI__builtin_neon_vld4_v:
1450 case NEON::BI__builtin_neon_vld4q_v:
1451 case NEON::BI__builtin_neon_vld2_dup_v:
1452 case NEON::BI__builtin_neon_vld2q_dup_v:
1453 case NEON::BI__builtin_neon_vld3_dup_v:
1454 case NEON::BI__builtin_neon_vld3q_dup_v:
1455 case NEON::BI__builtin_neon_vld4_dup_v:
1456 case NEON::BI__builtin_neon_vld4q_dup_v: {
1457 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1458 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1459 Value *Align = getAlignmentValue32(PtrOp1);
1460 Ops[1] = Builder.CreateCall(Callee: F, Args: {Ops[1], Align}, Name: NameHint);
1461 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
1462 }
1463 case NEON::BI__builtin_neon_vld1_dup_v:
1464 case NEON::BI__builtin_neon_vld1q_dup_v: {
1465 Value *V = PoisonValue::get(T: Ty);
1466 PtrOp0 = PtrOp0.withElementType(ElemTy: VTy->getElementType());
1467 LoadInst *Ld = Builder.CreateLoad(Addr: PtrOp0);
1468 llvm::Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
1469 Ops[0] = Builder.CreateInsertElement(Vec: V, NewElt: Ld, Idx: CI);
1470 return EmitNeonSplat(V: Ops[0], C: CI);
1471 }
1472 case NEON::BI__builtin_neon_vld2_lane_v:
1473 case NEON::BI__builtin_neon_vld2q_lane_v:
1474 case NEON::BI__builtin_neon_vld3_lane_v:
1475 case NEON::BI__builtin_neon_vld3q_lane_v:
1476 case NEON::BI__builtin_neon_vld4_lane_v:
1477 case NEON::BI__builtin_neon_vld4q_lane_v: {
1478 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1479 Function *F = CGM.getIntrinsic(IID: LLVMIntrinsic, Tys);
1480 for (unsigned I = 2; I < Ops.size() - 1; ++I)
1481 Ops[I] = Builder.CreateBitCast(V: Ops[I], DestTy: Ty);
1482 Ops.push_back(Elt: getAlignmentValue32(PtrOp1));
1483 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: NameHint);
1484 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
1485 }
1486 case NEON::BI__builtin_neon_vmovl_v: {
1487 llvm::FixedVectorType *DTy =
1488 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1489 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DTy);
1490 if (Usgn)
1491 return Builder.CreateZExt(V: Ops[0], DestTy: Ty, Name: "vmovl");
1492 return Builder.CreateSExt(V: Ops[0], DestTy: Ty, Name: "vmovl");
1493 }
1494 case NEON::BI__builtin_neon_vmovn_v: {
1495 llvm::FixedVectorType *QTy =
1496 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1497 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: QTy);
1498 return Builder.CreateTrunc(V: Ops[0], DestTy: Ty, Name: "vmovn");
1499 }
1500 case NEON::BI__builtin_neon_vmull_v:
1501 // FIXME: the integer vmull operations could be emitted in terms of pure
1502 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
1503 // hoisting the exts outside loops. Until global ISel comes along that can
1504 // see through such movement this leads to bad CodeGen. So we need an
1505 // intrinsic for now.
1506 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
1507 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
1508 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmull");
1509 case NEON::BI__builtin_neon_vpadal_v:
1510 case NEON::BI__builtin_neon_vpadalq_v: {
1511 // The source operand type has twice as many elements of half the size.
1512 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1513 llvm::Type *EltTy =
1514 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: EltBits / 2);
1515 auto *NarrowTy =
1516 llvm::FixedVectorType::get(ElementType: EltTy, NumElts: VTy->getNumElements() * 2);
1517 llvm::Type *Tys[2] = { Ty, NarrowTy };
1518 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: NameHint);
1519 }
1520 case NEON::BI__builtin_neon_vpaddl_v:
1521 case NEON::BI__builtin_neon_vpaddlq_v: {
1522 // The source operand type has twice as many elements of half the size.
1523 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1524 llvm::Type *EltTy = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: EltBits / 2);
1525 auto *NarrowTy =
1526 llvm::FixedVectorType::get(ElementType: EltTy, NumElts: VTy->getNumElements() * 2);
1527 llvm::Type *Tys[2] = { Ty, NarrowTy };
1528 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vpaddl");
1529 }
1530 case NEON::BI__builtin_neon_vqdmlal_v:
1531 case NEON::BI__builtin_neon_vqdmlsl_v: {
1532 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
1533 Ops[1] =
1534 EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys: Ty), Ops&: MulOps, name: "vqdmlal");
1535 Ops.resize(N: 2);
1536 return EmitNeonCall(F: CGM.getIntrinsic(IID: AltLLVMIntrinsic, Tys: Ty), Ops, name: NameHint);
1537 }
1538 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
1539 case NEON::BI__builtin_neon_vqdmulh_lane_v:
1540 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
1541 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
1542 auto *RTy = cast<llvm::FixedVectorType>(Val: Ty);
1543 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
1544 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
1545 RTy = llvm::FixedVectorType::get(ElementType: RTy->getElementType(),
1546 NumElts: RTy->getNumElements() * 2);
1547 llvm::Type *Tys[2] = {
1548 RTy, GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
1549 /*isQuad*/ false))};
1550 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: NameHint);
1551 }
1552 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
1553 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
1554 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
1555 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
1556 llvm::Type *Tys[2] = {
1557 Ty, GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
1558 /*isQuad*/ true))};
1559 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: NameHint);
1560 }
1561 case NEON::BI__builtin_neon_vqshl_n_v:
1562 case NEON::BI__builtin_neon_vqshlq_n_v:
1563 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshl_n",
1564 shift: 1, rightshift: false);
1565 case NEON::BI__builtin_neon_vqshlu_n_v:
1566 case NEON::BI__builtin_neon_vqshluq_n_v:
1567 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshlu_n",
1568 shift: 1, rightshift: false);
1569 case NEON::BI__builtin_neon_vrecpe_v:
1570 case NEON::BI__builtin_neon_vrecpeq_v:
1571 case NEON::BI__builtin_neon_vrsqrte_v:
1572 case NEON::BI__builtin_neon_vrsqrteq_v:
1573 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
1574 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: NameHint);
1575 case NEON::BI__builtin_neon_vrndi_v:
1576 case NEON::BI__builtin_neon_vrndiq_v:
1577 Int = Builder.getIsFPConstrained()
1578 ? Intrinsic::experimental_constrained_nearbyint
1579 : Intrinsic::nearbyint;
1580 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: NameHint);
1581 case NEON::BI__builtin_neon_vrshr_n_v:
1582 case NEON::BI__builtin_neon_vrshrq_n_v:
1583 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrshr_n",
1584 shift: 1, rightshift: true);
1585 case NEON::BI__builtin_neon_vsha512hq_u64:
1586 case NEON::BI__builtin_neon_vsha512h2q_u64:
1587 case NEON::BI__builtin_neon_vsha512su0q_u64:
1588 case NEON::BI__builtin_neon_vsha512su1q_u64: {
1589 Function *F = CGM.getIntrinsic(IID: Int);
1590 return EmitNeonCall(F, Ops, name: "");
1591 }
1592 case NEON::BI__builtin_neon_vshl_n_v:
1593 case NEON::BI__builtin_neon_vshlq_n_v:
1594 Ops[1] = EmitNeonShiftVector(V: Ops[1], Ty, neg: false);
1595 return Builder.CreateShl(LHS: Builder.CreateBitCast(V: Ops[0],DestTy: Ty), RHS: Ops[1],
1596 Name: "vshl_n");
1597 case NEON::BI__builtin_neon_vshll_n_v: {
1598 llvm::FixedVectorType *SrcTy =
1599 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1600 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1601 if (Usgn)
1602 Ops[0] = Builder.CreateZExt(V: Ops[0], DestTy: VTy);
1603 else
1604 Ops[0] = Builder.CreateSExt(V: Ops[0], DestTy: VTy);
1605 Ops[1] = EmitNeonShiftVector(V: Ops[1], Ty: VTy, neg: false);
1606 return Builder.CreateShl(LHS: Ops[0], RHS: Ops[1], Name: "vshll_n");
1607 }
1608 case NEON::BI__builtin_neon_vshrn_n_v: {
1609 llvm::FixedVectorType *SrcTy =
1610 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1611 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1612 Ops[1] = EmitNeonShiftVector(V: Ops[1], Ty: SrcTy, neg: false);
1613 if (Usgn)
1614 Ops[0] = Builder.CreateLShr(LHS: Ops[0], RHS: Ops[1]);
1615 else
1616 Ops[0] = Builder.CreateAShr(LHS: Ops[0], RHS: Ops[1]);
1617 return Builder.CreateTrunc(V: Ops[0], DestTy: Ty, Name: "vshrn_n");
1618 }
1619 case NEON::BI__builtin_neon_vshr_n_v:
1620 case NEON::BI__builtin_neon_vshrq_n_v:
1621 return EmitNeonRShiftImm(Vec: Ops[0], Shift: Ops[1], Ty, usgn: Usgn, name: "vshr_n");
1622 case NEON::BI__builtin_neon_vst1_v:
1623 case NEON::BI__builtin_neon_vst1q_v:
1624 case NEON::BI__builtin_neon_vst2_v:
1625 case NEON::BI__builtin_neon_vst2q_v:
1626 case NEON::BI__builtin_neon_vst3_v:
1627 case NEON::BI__builtin_neon_vst3q_v:
1628 case NEON::BI__builtin_neon_vst4_v:
1629 case NEON::BI__builtin_neon_vst4q_v:
1630 case NEON::BI__builtin_neon_vst2_lane_v:
1631 case NEON::BI__builtin_neon_vst2q_lane_v:
1632 case NEON::BI__builtin_neon_vst3_lane_v:
1633 case NEON::BI__builtin_neon_vst3q_lane_v:
1634 case NEON::BI__builtin_neon_vst4_lane_v:
1635 case NEON::BI__builtin_neon_vst4q_lane_v: {
1636 llvm::Type *Tys[] = {Int8PtrTy, Ty};
1637 Ops.push_back(Elt: getAlignmentValue32(PtrOp0));
1638 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "");
1639 }
1640 case NEON::BI__builtin_neon_vsm3partw1q_u32:
1641 case NEON::BI__builtin_neon_vsm3partw2q_u32:
1642 case NEON::BI__builtin_neon_vsm3ss1q_u32:
1643 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
1644 case NEON::BI__builtin_neon_vsm4eq_u32: {
1645 Function *F = CGM.getIntrinsic(IID: Int);
1646 return EmitNeonCall(F, Ops, name: "");
1647 }
1648 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
1649 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
1650 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
1651 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
1652 Function *F = CGM.getIntrinsic(IID: Int);
1653 Ops[3] = Builder.CreateZExt(V: Ops[3], DestTy: Int64Ty);
1654 return EmitNeonCall(F, Ops, name: "");
1655 }
1656 case NEON::BI__builtin_neon_vst1_x2_v:
1657 case NEON::BI__builtin_neon_vst1q_x2_v:
1658 case NEON::BI__builtin_neon_vst1_x3_v:
1659 case NEON::BI__builtin_neon_vst1q_x3_v:
1660 case NEON::BI__builtin_neon_vst1_x4_v:
1661 case NEON::BI__builtin_neon_vst1q_x4_v: {
1662 // TODO: Currently in AArch32 mode the pointer operand comes first, whereas
1663 // in AArch64 it comes last. We may want to stick to one or another.
1664 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
1665 Arch == llvm::Triple::aarch64_32) {
1666 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
1667 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
1668 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "");
1669 }
1670 llvm::Type *Tys[2] = {DefaultPtrTy, VTy};
1671 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "");
1672 }
1673 case NEON::BI__builtin_neon_vsubhn_v: {
1674 llvm::FixedVectorType *SrcTy =
1675 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1676
1677 // %sum = add <4 x i32> %lhs, %rhs
1678 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: SrcTy);
1679 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: SrcTy);
1680 Ops[0] = Builder.CreateSub(LHS: Ops[0], RHS: Ops[1], Name: "vsubhn");
1681
1682 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
1683 Constant *ShiftAmt =
1684 ConstantInt::get(Ty: SrcTy, V: SrcTy->getScalarSizeInBits() / 2);
1685 Ops[0] = Builder.CreateLShr(LHS: Ops[0], RHS: ShiftAmt, Name: "vsubhn");
1686
1687 // %res = trunc <4 x i32> %high to <4 x i16>
1688 return Builder.CreateTrunc(V: Ops[0], DestTy: VTy, Name: "vsubhn");
1689 }
1690 case NEON::BI__builtin_neon_vtrn_v:
1691 case NEON::BI__builtin_neon_vtrnq_v: {
1692 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1693 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1694 Value *SV = nullptr;
1695
1696 for (unsigned vi = 0; vi != 2; ++vi) {
1697 SmallVector<int, 16> Indices;
1698 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1699 Indices.push_back(Elt: i+vi);
1700 Indices.push_back(Elt: i+e+vi);
1701 }
1702 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
1703 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vtrn");
1704 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
1705 }
1706 return SV;
1707 }
1708 case NEON::BI__builtin_neon_vtst_v:
1709 case NEON::BI__builtin_neon_vtstq_v: {
1710 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
1711 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1712 Ops[0] = Builder.CreateAnd(LHS: Ops[0], RHS: Ops[1]);
1713 Ops[0] = Builder.CreateICmp(P: ICmpInst::ICMP_NE, LHS: Ops[0],
1714 RHS: ConstantAggregateZero::get(Ty));
1715 return Builder.CreateSExt(V: Ops[0], DestTy: Ty, Name: "vtst");
1716 }
1717 case NEON::BI__builtin_neon_vuzp_v:
1718 case NEON::BI__builtin_neon_vuzpq_v: {
1719 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1720 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1721 Value *SV = nullptr;
1722
1723 for (unsigned vi = 0; vi != 2; ++vi) {
1724 SmallVector<int, 16> Indices;
1725 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1726 Indices.push_back(Elt: 2*i+vi);
1727
1728 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
1729 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vuzp");
1730 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
1731 }
1732 return SV;
1733 }
1734 case NEON::BI__builtin_neon_vxarq_u64: {
1735 Function *F = CGM.getIntrinsic(IID: Int);
1736 Ops[2] = Builder.CreateZExt(V: Ops[2], DestTy: Int64Ty);
1737 return EmitNeonCall(F, Ops, name: "");
1738 }
1739 case NEON::BI__builtin_neon_vzip_v:
1740 case NEON::BI__builtin_neon_vzipq_v: {
1741 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
1742 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
1743 Value *SV = nullptr;
1744
1745 for (unsigned vi = 0; vi != 2; ++vi) {
1746 SmallVector<int, 16> Indices;
1747 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1748 Indices.push_back(Elt: (i + vi*e) >> 1);
1749 Indices.push_back(Elt: ((i + vi*e) >> 1)+e);
1750 }
1751 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
1752 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vzip");
1753 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
1754 }
1755 return SV;
1756 }
1757 case NEON::BI__builtin_neon_vdot_s32:
1758 case NEON::BI__builtin_neon_vdot_u32:
1759 case NEON::BI__builtin_neon_vdotq_s32:
1760 case NEON::BI__builtin_neon_vdotq_u32: {
1761 auto *InputTy =
1762 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1763 llvm::Type *Tys[2] = { Ty, InputTy };
1764 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vdot");
1765 }
1766 case NEON::BI__builtin_neon_vfmlal_low_f16:
1767 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
1768 auto *InputTy =
1769 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1770 llvm::Type *Tys[2] = { Ty, InputTy };
1771 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlal_low");
1772 }
1773 case NEON::BI__builtin_neon_vfmlsl_low_f16:
1774 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
1775 auto *InputTy =
1776 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1777 llvm::Type *Tys[2] = { Ty, InputTy };
1778 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlsl_low");
1779 }
1780 case NEON::BI__builtin_neon_vfmlal_high_f16:
1781 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
1782 auto *InputTy =
1783 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1784 llvm::Type *Tys[2] = { Ty, InputTy };
1785 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlal_high");
1786 }
1787 case NEON::BI__builtin_neon_vfmlsl_high_f16:
1788 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
1789 auto *InputTy =
1790 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1791 llvm::Type *Tys[2] = { Ty, InputTy };
1792 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vfmlsl_high");
1793 }
1794 case NEON::BI__builtin_neon_vmmlaq_s32:
1795 case NEON::BI__builtin_neon_vmmlaq_u32: {
1796 auto *InputTy =
1797 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1798 llvm::Type *Tys[2] = { Ty, InputTy };
1799 return EmitNeonCall(F: CGM.getIntrinsic(IID: LLVMIntrinsic, Tys), Ops, name: "vmmla");
1800 }
1801 case NEON::BI__builtin_neon_vmmlaq_f16:
1802 case NEON::BI__builtin_neon_vmmlaq_f32_f16: {
1803 auto *InputTy =
1804 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1805 llvm::Type *Tys[2] = {Ty, InputTy};
1806 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fmmla");
1807 }
1808 case NEON::BI__builtin_neon_vusmmlaq_s32: {
1809 auto *InputTy =
1810 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1811 llvm::Type *Tys[2] = { Ty, InputTy };
1812 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vusmmla");
1813 }
1814 case NEON::BI__builtin_neon_vusdot_s32:
1815 case NEON::BI__builtin_neon_vusdotq_s32: {
1816 auto *InputTy =
1817 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: Ty->getPrimitiveSizeInBits() / 8);
1818 llvm::Type *Tys[2] = { Ty, InputTy };
1819 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vusdot");
1820 }
1821 case NEON::BI__builtin_neon_vbfdot_f32:
1822 case NEON::BI__builtin_neon_vbfdotq_f32: {
1823 llvm::Type *InputTy =
1824 llvm::FixedVectorType::get(ElementType: BFloatTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
1825 llvm::Type *Tys[2] = { Ty, InputTy };
1826 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vbfdot");
1827 }
1828 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
1829 llvm::Type *Tys[1] = { Ty };
1830 Function *F = CGM.getIntrinsic(IID: Int, Tys);
1831 return EmitNeonCall(F, Ops, name: "vcvtfp2bf");
1832 }
1833
1834 }
1835
1836 assert(Int && "Expected valid intrinsic number");
1837
1838 // Determine the type(s) of this overloaded AArch64 intrinsic.
1839 Function *F = LookupNeonLLVMIntrinsic(IntrinsicID: Int, Modifier, ArgType: Ty, E);
1840
1841 Value *Result = EmitNeonCall(F, Ops, name: NameHint);
1842 llvm::Type *ResultType = ConvertType(T: E->getType());
1843 // AArch64 intrinsic one-element vector type cast to
1844 // scalar type expected by the builtin
1845 return Builder.CreateBitCast(V: Result, DestTy: ResultType, Name: NameHint);
1846}
1847
1848Value *
1849CodeGenFunction::EmitAArch64CompareBuiltinExpr(Value *Op, llvm::Type *Ty,
1850 const CmpInst::Predicate Pred,
1851 const Twine &Name) {
1852
1853 if (isa<FixedVectorType>(Val: Ty)) {
1854 // Vector types are cast to i8 vectors. Recover original type.
1855 Op = Builder.CreateBitCast(V: Op, DestTy: Ty);
1856 }
1857
1858 Constant *zero = Constant::getNullValue(Ty: Op->getType());
1859
1860 if (CmpInst::isFPPredicate(P: Pred)) {
1861 if (Pred == CmpInst::FCMP_OEQ)
1862 Op = Builder.CreateFCmp(P: Pred, LHS: Op, RHS: zero);
1863 else
1864 Op = Builder.CreateFCmpS(P: Pred, LHS: Op, RHS: zero);
1865 } else {
1866 Op = Builder.CreateICmp(P: Pred, LHS: Op, RHS: zero);
1867 }
1868
1869 llvm::Type *ResTy = Ty;
1870 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty))
1871 ResTy = FixedVectorType::get(
1872 ElementType: IntegerType::get(C&: getLLVMContext(), NumBits: VTy->getScalarSizeInBits()),
1873 NumElts: VTy->getNumElements());
1874
1875 return Builder.CreateSExt(V: Op, DestTy: ResTy, Name);
1876}
1877
1878static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
1879 Value *ExtOp, Value *IndexOp,
1880 llvm::Type *ResTy, unsigned IntID,
1881 const char *Name) {
1882 SmallVector<Value *, 2> TblOps;
1883 if (ExtOp)
1884 TblOps.push_back(Elt: ExtOp);
1885
1886 // Build a vector containing sequential number like (0, 1, 2, ..., 15)
1887 SmallVector<int, 16> Indices;
1888 auto *TblTy = cast<llvm::FixedVectorType>(Val: Ops[0]->getType());
1889 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
1890 Indices.push_back(Elt: 2*i);
1891 Indices.push_back(Elt: 2*i+1);
1892 }
1893
1894 int PairPos = 0, End = Ops.size() - 1;
1895 while (PairPos < End) {
1896 TblOps.push_back(Elt: CGF.Builder.CreateShuffleVector(V1: Ops[PairPos],
1897 V2: Ops[PairPos+1], Mask: Indices,
1898 Name));
1899 PairPos += 2;
1900 }
1901
1902 // If there's an odd number of 64-bit lookup table, fill the high 64-bit
1903 // of the 128-bit lookup table with zero.
1904 if (PairPos == End) {
1905 Value *ZeroTbl = ConstantAggregateZero::get(Ty: TblTy);
1906 TblOps.push_back(Elt: CGF.Builder.CreateShuffleVector(V1: Ops[PairPos],
1907 V2: ZeroTbl, Mask: Indices, Name));
1908 }
1909
1910 Function *TblF;
1911 TblOps.push_back(Elt: IndexOp);
1912 TblF = CGF.CGM.getIntrinsic(IID: IntID, Tys: ResTy);
1913
1914 return CGF.EmitNeonCall(F: TblF, Ops&: TblOps, name: Name);
1915}
1916
1917Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
1918 unsigned Value;
1919 switch (BuiltinID) {
1920 default:
1921 return nullptr;
1922 case clang::ARM::BI__builtin_arm_nop:
1923 Value = 0;
1924 break;
1925 case clang::ARM::BI__builtin_arm_yield:
1926 case clang::ARM::BI__yield:
1927 Value = 1;
1928 break;
1929 case clang::ARM::BI__builtin_arm_wfe:
1930 case clang::ARM::BI__wfe:
1931 Value = 2;
1932 break;
1933 case clang::ARM::BI__builtin_arm_wfi:
1934 case clang::ARM::BI__wfi:
1935 Value = 3;
1936 break;
1937 case clang::ARM::BI__builtin_arm_sev:
1938 case clang::ARM::BI__sev:
1939 Value = 4;
1940 break;
1941 case clang::ARM::BI__builtin_arm_sevl:
1942 case clang::ARM::BI__sevl:
1943 Value = 5;
1944 break;
1945 }
1946
1947 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_hint),
1948 Args: llvm::ConstantInt::get(Ty: Int32Ty, V: Value));
1949}
1950
1951enum SpecialRegisterAccessKind {
1952 NormalRead,
1953 VolatileRead,
1954 Write,
1955};
1956
1957// Generates the IR for the read/write special register builtin,
1958// ValueType is the type of the value that is to be written or read,
1959// RegisterType is the type of the register being written to or read from.
1960static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF,
1961 const CallExpr *E,
1962 llvm::Type *RegisterType,
1963 llvm::Type *ValueType,
1964 SpecialRegisterAccessKind AccessKind,
1965 StringRef SysReg = "") {
1966 // write and register intrinsics only support 32, 64 and 128 bit operations.
1967 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
1968 RegisterType->isIntegerTy(128)) &&
1969 "Unsupported size for register.");
1970
1971 CodeGen::CGBuilderTy &Builder = CGF.Builder;
1972 CodeGen::CodeGenModule &CGM = CGF.CGM;
1973 LLVMContext &Context = CGM.getLLVMContext();
1974
1975 if (SysReg.empty()) {
1976 const Expr *SysRegStrExpr = E->getArg(Arg: 0)->IgnoreParenCasts();
1977 SysReg = cast<clang::StringLiteral>(Val: SysRegStrExpr)->getString();
1978 }
1979
1980 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, Str: SysReg) };
1981 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
1982 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
1983
1984 llvm::Type *Types[] = { RegisterType };
1985
1986 bool MixedTypes = RegisterType->isIntegerTy(BitWidth: 64) && ValueType->isIntegerTy(BitWidth: 32);
1987 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
1988 && "Can't fit 64-bit value in 32-bit register");
1989
1990 if (AccessKind != Write) {
1991 assert(AccessKind == NormalRead || AccessKind == VolatileRead);
1992 llvm::Function *F = CGM.getIntrinsic(
1993 IID: AccessKind == VolatileRead ? Intrinsic::read_volatile_register
1994 : Intrinsic::read_register,
1995 Tys: Types);
1996 llvm::Value *Call = Builder.CreateCall(Callee: F, Args: Metadata);
1997
1998 if (MixedTypes)
1999 // Read into 64 bit register and then truncate result to 32 bit.
2000 return Builder.CreateTrunc(V: Call, DestTy: ValueType);
2001
2002 if (ValueType->isPointerTy())
2003 // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*).
2004 return Builder.CreateIntToPtr(V: Call, DestTy: ValueType);
2005
2006 return Call;
2007 }
2008
2009 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::write_register, Tys: Types);
2010 llvm::Value *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
2011 if (MixedTypes) {
2012 // Extend 32 bit write value to 64 bit to pass to write.
2013 ArgValue = Builder.CreateZExt(V: ArgValue, DestTy: RegisterType);
2014 return Builder.CreateCall(Callee: F, Args: { Metadata, ArgValue });
2015 }
2016
2017 if (ValueType->isPointerTy()) {
2018 // Have VoidPtrTy ArgValue but want to return an i32/i64.
2019 ArgValue = Builder.CreatePtrToInt(V: ArgValue, DestTy: RegisterType);
2020 return Builder.CreateCall(Callee: F, Args: { Metadata, ArgValue });
2021 }
2022
2023 return Builder.CreateCall(Callee: F, Args: { Metadata, ArgValue });
2024}
2025
2026static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
2027 const CallExpr *E) {
2028 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2029 CodeGen::CodeGenModule &CGM = CGF.CGM;
2030 SmallVector<llvm::Value *, 4> Ops;
2031
2032 auto getIntArg = [&](unsigned ArgNo) {
2033 Expr::EvalResult Result;
2034 if (!E->getArg(Arg: ArgNo)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
2035 llvm_unreachable("Expected constant argument to range prefetch.");
2036 return Result.Val.getInt().getExtValue();
2037 };
2038
2039 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 0))); /*Addr*/
2040 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 1))); /*Access Kind*/
2041 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 2))); /*Policy*/
2042
2043 if (BuiltinID == clang::AArch64::BI__builtin_arm_range_prefetch_x) {
2044 auto Length = getIntArg(3);
2045 auto Count = getIntArg(4) - 1;
2046 auto Stride = getIntArg(5);
2047 auto Distance = getIntArg(6);
2048
2049 // Map ReuseDistance given in bytes to four bits representing decreasing
2050 // powers of two in the range 512MiB (0b0001) to 32KiB (0b1111). Values
2051 // are rounded up to the nearest power of 2, starting at 32KiB. Any value
2052 // over the maximum is represented by 0 (distance not known).
2053 if (Distance > 0) {
2054 Distance = llvm::Log2_32_Ceil(Value: Distance);
2055 if (Distance < 15)
2056 Distance = 15;
2057 else if (Distance > 29)
2058 Distance = 0;
2059 else
2060 Distance = 30 - Distance;
2061 }
2062
2063 uint64_t Mask22 = (1ULL << 22) - 1;
2064 uint64_t Mask16 = (1ULL << 16) - 1;
2065 uint64_t Metadata = (Distance << 60) | ((Stride & Mask22) << 38) |
2066 ((Count & Mask16) << 22) | (Length & Mask22);
2067
2068 Ops.push_back(Elt: llvm::ConstantInt::get(Ty: Builder.getInt64Ty(), V: Metadata));
2069 } else
2070 Ops.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 3)));
2071
2072 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_range_prefetch),
2073 Args: Ops);
2074}
2075
2076static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E,
2077 Instruction *Atomic, unsigned PtrOperand) {
2078 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2079 LLVMContext &Ctx = CGF.CGM.getLLVMContext();
2080 Expr::EvalResult Result;
2081 if (!E->getArg(Arg: 3)->EvaluateAsInt(Result, Ctx: CGF.getContext()))
2082 llvm_unreachable(
2083 "Expected integer hint argument to atomic operation with hint.");
2084 unsigned HintArg = Result.Val.getInt().getExtValue();
2085
2086 auto MemoryHint = toAArch64MemoryHint(I: HintArg);
2087 if (MemoryHint == AArch64MemoryHint::NONE ||
2088 (isa<AtomicRMWInst>(Val: Atomic) && MemoryHint != AArch64MemoryHint::SHUH &&
2089 MemoryHint != AArch64MemoryHint::SHUH_PH))
2090 return;
2091
2092 MDNode *MemHint = MDNode::get(
2093 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "aarch64.mem_hint"),
2094 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: HintArg))});
2095 MDNode *HintNode = MDNode::get(
2096 Context&: Ctx,
2097 MDs: {llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: PtrOperand)), MemHint});
2098 Atomic->setMetadata(KindID: llvm::LLVMContext::MD_mem_cache_hint, Node: HintNode);
2099}
2100
2101static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
2102 unsigned BuiltinID,
2103 const CallExpr *E) {
2104 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2105 CodeGen::CodeGenModule &CGM = CGF.CGM;
2106 Expr::EvalResult Result;
2107 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
2108 llvm_unreachable(
2109 "Expected integer policy argument to atomic store with hint.");
2110
2111 const Expr *Ptr = E->getArg(Arg: 0);
2112 Address Addr = CGF.EmitPointerWithAlignment(Addr: Ptr);
2113 Addr = Addr.withElementType(
2114 ElemTy: CGF.ConvertTypeForMem(T: Ptr->getType()->getPointeeType()));
2115
2116 const Expr *Data = E->getArg(Arg: 1);
2117 Value *DataVal = CGF.EmitToMemory(Value: CGF.EmitScalarExpr(E: Data), Ty: Data->getType());
2118
2119 StoreInst *Store = Builder.CreateStore(Val: DataVal, Addr);
2120 Store->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
2121
2122 AtomicOrdering Ordering;
2123 unsigned OrderingArg = Result.Val.getInt().getExtValue();
2124 assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
2125
2126 switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
2127 default:
2128 llvm_unreachable("Unsupported atomic ordering found.");
2129 case AtomicOrderingCABI::relaxed:
2130 Ordering = AtomicOrdering::Monotonic;
2131 break;
2132 case AtomicOrderingCABI::release:
2133 Ordering = AtomicOrdering::Release;
2134 break;
2135 case AtomicOrderingCABI::seq_cst:
2136 Ordering = AtomicOrdering::SequentiallyConsistent;
2137 break;
2138 }
2139 Store->setAtomic(Ordering);
2140 AttachAtomicHintMetadata(CGF, E, Atomic: Store, PtrOperand: 1);
2141 return Store;
2142}
2143
2144static Value *EmitAtomicFetchWithHintBuiltin(CodeGenFunction &CGF,
2145 const CallExpr *E,
2146 AtomicRMWInst::BinOp Op) {
2147 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2148 Expr::EvalResult Result;
2149 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result, Ctx: CGF.getContext()))
2150 llvm_unreachable(
2151 "Expected integer policy argument to atomic fetch with hint.");
2152
2153 AtomicOrdering Ordering;
2154 unsigned OrderingArg = Result.Val.getInt().getExtValue();
2155 assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
2156
2157 switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
2158 case AtomicOrderingCABI::relaxed:
2159 Ordering = AtomicOrdering::Monotonic;
2160 break;
2161 case AtomicOrderingCABI::consume:
2162 case AtomicOrderingCABI::acquire:
2163 Ordering = AtomicOrdering::Acquire;
2164 break;
2165 case AtomicOrderingCABI::release:
2166 Ordering = AtomicOrdering::Release;
2167 break;
2168 case AtomicOrderingCABI::acq_rel:
2169 Ordering = AtomicOrdering::AcquireRelease;
2170 break;
2171 case AtomicOrderingCABI::seq_cst:
2172 Ordering = AtomicOrdering::SequentiallyConsistent;
2173 break;
2174 }
2175
2176 const Expr *Ptr = E->getArg(Arg: 0);
2177 Address Addr = CGF.EmitPointerWithAlignment(Addr: Ptr);
2178 Addr = Addr.withElementType(
2179 ElemTy: CGF.ConvertTypeForMem(T: Ptr->getType()->getPointeeType()));
2180 const Expr *Data = E->getArg(Arg: 1);
2181 Value *DataVal = CGF.EmitToMemory(Value: CGF.EmitScalarExpr(E: Data), Ty: Data->getType());
2182
2183 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(Op, Addr, Val: DataVal, Ordering);
2184 RMW->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
2185 AttachAtomicHintMetadata(CGF, E, Atomic: RMW, PtrOperand: 0);
2186 return CGF.EmitFromMemory(Value: RMW, Ty: E->getType());
2187}
2188
2189Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
2190 const CallExpr *E,
2191 ReturnValueSlot ReturnValue,
2192 llvm::Triple::ArchType Arch) {
2193 if (auto Hint = GetValueForARMHint(BuiltinID))
2194 return Hint;
2195
2196 if (BuiltinID == clang::ARM::BI__emit) {
2197 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
2198 llvm::FunctionType *FTy =
2199 llvm::FunctionType::get(Result: VoidTy, /*Variadic=*/isVarArg: false);
2200
2201 Expr::EvalResult Result;
2202 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
2203 llvm_unreachable("Sema will ensure that the parameter is constant");
2204
2205 llvm::APSInt Value = Result.Val.getInt();
2206 uint64_t ZExtValue = Value.zextOrTrunc(width: IsThumb ? 16 : 32).getZExtValue();
2207
2208 llvm::InlineAsm *Emit =
2209 IsThumb ? InlineAsm::get(Ty: FTy, AsmString: ".inst.n 0x" + utohexstr(X: ZExtValue), Constraints: "",
2210 /*hasSideEffects=*/true)
2211 : InlineAsm::get(Ty: FTy, AsmString: ".inst 0x" + utohexstr(X: ZExtValue), Constraints: "",
2212 /*hasSideEffects=*/true);
2213
2214 return Builder.CreateCall(Callee: Emit);
2215 }
2216
2217 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
2218 Value *Option = EmitScalarExpr(E: E->getArg(Arg: 0));
2219 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_dbg), Args: Option);
2220 }
2221
2222 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
2223 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
2224 Value *RW = EmitScalarExpr(E: E->getArg(Arg: 1));
2225 Value *IsData = EmitScalarExpr(E: E->getArg(Arg: 2));
2226
2227 // Locality is not supported on ARM target
2228 Value *Locality = llvm::ConstantInt::get(Ty: Int32Ty, V: 3);
2229
2230 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
2231 return Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, IsData});
2232 }
2233
2234 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
2235 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2236 return Builder.CreateCall(
2237 Callee: CGM.getIntrinsic(IID: Intrinsic::bitreverse, Tys: Arg->getType()), Args: Arg, Name: "rbit");
2238 }
2239
2240 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
2241 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
2242 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2243 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: Arg->getType());
2244 Value *Res = Builder.CreateCall(Callee: F, Args: {Arg, Builder.getInt1(V: false)});
2245 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
2246 Res = Builder.CreateTrunc(V: Res, DestTy: Builder.getInt32Ty());
2247 return Res;
2248 }
2249
2250
2251 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
2252 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2253 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_cls), Args: Arg, Name: "cls");
2254 }
2255 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
2256 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2257 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_cls64), Args: Arg,
2258 Name: "cls");
2259 }
2260
2261 if (BuiltinID == clang::ARM::BI__clear_cache) {
2262 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
2263 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
2264 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache, Tys: {CGM.DefaultPtrTy});
2265 return Builder.CreateCall(Callee: F, Args: {Begin, End});
2266 }
2267
2268 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
2269 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
2270 Function *F;
2271
2272 switch (BuiltinID) {
2273 default: llvm_unreachable("unexpected builtin");
2274 case clang::ARM::BI__builtin_arm_mcrr:
2275 F = CGM.getIntrinsic(IID: Intrinsic::arm_mcrr);
2276 break;
2277 case clang::ARM::BI__builtin_arm_mcrr2:
2278 F = CGM.getIntrinsic(IID: Intrinsic::arm_mcrr2);
2279 break;
2280 }
2281
2282 // MCRR{2} instruction has 5 operands but
2283 // the intrinsic has 4 because Rt and Rt2
2284 // are represented as a single unsigned 64
2285 // bit integer in the intrinsic definition
2286 // but internally it's represented as 2 32
2287 // bit integers.
2288
2289 Value *Coproc = EmitScalarExpr(E: E->getArg(Arg: 0));
2290 Value *Opc1 = EmitScalarExpr(E: E->getArg(Arg: 1));
2291 Value *RtAndRt2 = EmitScalarExpr(E: E->getArg(Arg: 2));
2292 Value *CRm = EmitScalarExpr(E: E->getArg(Arg: 3));
2293
2294 Value *C1 = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2295 Value *Rt = Builder.CreateTruncOrBitCast(V: RtAndRt2, DestTy: Int32Ty);
2296 Value *Rt2 = Builder.CreateLShr(LHS: RtAndRt2, RHS: C1);
2297 Rt2 = Builder.CreateTruncOrBitCast(V: Rt2, DestTy: Int32Ty);
2298
2299 return Builder.CreateCall(Callee: F, Args: {Coproc, Opc1, Rt, Rt2, CRm});
2300 }
2301
2302 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
2303 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
2304 Function *F;
2305
2306 switch (BuiltinID) {
2307 default: llvm_unreachable("unexpected builtin");
2308 case clang::ARM::BI__builtin_arm_mrrc:
2309 F = CGM.getIntrinsic(IID: Intrinsic::arm_mrrc);
2310 break;
2311 case clang::ARM::BI__builtin_arm_mrrc2:
2312 F = CGM.getIntrinsic(IID: Intrinsic::arm_mrrc2);
2313 break;
2314 }
2315
2316 Value *Coproc = EmitScalarExpr(E: E->getArg(Arg: 0));
2317 Value *Opc1 = EmitScalarExpr(E: E->getArg(Arg: 1));
2318 Value *CRm = EmitScalarExpr(E: E->getArg(Arg: 2));
2319 Value *RtAndRt2 = Builder.CreateCall(Callee: F, Args: {Coproc, Opc1, CRm});
2320
2321 // Returns an unsigned 64 bit integer, represented
2322 // as two 32 bit integers.
2323
2324 Value *Rt = Builder.CreateExtractValue(Agg: RtAndRt2, Idxs: 1);
2325 Value *Rt1 = Builder.CreateExtractValue(Agg: RtAndRt2, Idxs: 0);
2326 Rt = Builder.CreateZExt(V: Rt, DestTy: Int64Ty);
2327 Rt1 = Builder.CreateZExt(V: Rt1, DestTy: Int64Ty);
2328
2329 Value *ShiftCast = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2330 RtAndRt2 = Builder.CreateShl(LHS: Rt, RHS: ShiftCast, Name: "shl", HasNUW: true);
2331 RtAndRt2 = Builder.CreateOr(LHS: RtAndRt2, RHS: Rt1);
2332
2333 return Builder.CreateBitCast(V: RtAndRt2, DestTy: ConvertType(T: E->getType()));
2334 }
2335
2336 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
2337 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2338 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
2339 getContext().getTypeSize(T: E->getType()) == 64) ||
2340 BuiltinID == clang::ARM::BI__ldrexd) {
2341 Function *F;
2342
2343 switch (BuiltinID) {
2344 default: llvm_unreachable("unexpected builtin");
2345 case clang::ARM::BI__builtin_arm_ldaex:
2346 F = CGM.getIntrinsic(IID: Intrinsic::arm_ldaexd);
2347 break;
2348 case clang::ARM::BI__builtin_arm_ldrexd:
2349 case clang::ARM::BI__builtin_arm_ldrex:
2350 case clang::ARM::BI__ldrexd:
2351 F = CGM.getIntrinsic(IID: Intrinsic::arm_ldrexd);
2352 break;
2353 }
2354
2355 Value *LdPtr = EmitScalarExpr(E: E->getArg(Arg: 0));
2356 Value *Val = Builder.CreateCall(Callee: F, Args: LdPtr, Name: "ldrexd");
2357
2358 Value *Val0 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
2359 Value *Val1 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
2360 Val0 = Builder.CreateZExt(V: Val0, DestTy: Int64Ty);
2361 Val1 = Builder.CreateZExt(V: Val1, DestTy: Int64Ty);
2362
2363 Value *ShiftCst = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2364 Val = Builder.CreateShl(LHS: Val0, RHS: ShiftCst, Name: "shl", HasNUW: true /* nuw */);
2365 Val = Builder.CreateOr(LHS: Val, RHS: Val1);
2366 return Builder.CreateBitCast(V: Val, DestTy: ConvertType(T: E->getType()));
2367 }
2368
2369 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2370 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
2371 Value *LoadAddr = EmitScalarExpr(E: E->getArg(Arg: 0));
2372
2373 QualType Ty = E->getType();
2374 llvm::Type *RealResTy = ConvertType(T: Ty);
2375 llvm::Type *IntTy =
2376 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
2377
2378 Function *F = CGM.getIntrinsic(
2379 IID: BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
2380 : Intrinsic::arm_ldrex,
2381 Tys: DefaultPtrTy);
2382 CallInst *Val = Builder.CreateCall(Callee: F, Args: LoadAddr, Name: "ldrex");
2383 Val->addParamAttr(
2384 ArgNo: 0, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: IntTy));
2385
2386 if (RealResTy->isPointerTy())
2387 return Builder.CreateIntToPtr(V: Val, DestTy: RealResTy);
2388 else {
2389 llvm::Type *IntResTy = llvm::IntegerType::get(
2390 C&: getLLVMContext(), NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: RealResTy));
2391 return Builder.CreateBitCast(V: Builder.CreateTruncOrBitCast(V: Val, DestTy: IntResTy),
2392 DestTy: RealResTy);
2393 }
2394 }
2395
2396 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
2397 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
2398 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
2399 getContext().getTypeSize(T: E->getArg(Arg: 0)->getType()) == 64)) {
2400 Function *F = CGM.getIntrinsic(
2401 IID: BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
2402 : Intrinsic::arm_strexd);
2403 llvm::Type *STy = llvm::StructType::get(elt1: Int32Ty, elts: Int32Ty);
2404
2405 Address Tmp = CreateMemTempWithoutCast(T: E->getArg(Arg: 0)->getType());
2406 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
2407 Builder.CreateStore(Val, Addr: Tmp);
2408
2409 Address LdPtr = Tmp.withElementType(ElemTy: STy);
2410 Val = Builder.CreateLoad(Addr: LdPtr);
2411
2412 Value *Arg0 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
2413 Value *Arg1 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
2414 Value *StPtr = EmitScalarExpr(E: E->getArg(Arg: 1));
2415 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1, StPtr}, Name: "strexd");
2416 }
2417
2418 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
2419 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
2420 Value *StoreVal = EmitScalarExpr(E: E->getArg(Arg: 0));
2421 Value *StoreAddr = EmitScalarExpr(E: E->getArg(Arg: 1));
2422
2423 QualType Ty = E->getArg(Arg: 0)->getType();
2424 llvm::Type *StoreTy =
2425 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
2426
2427 if (StoreVal->getType()->isPointerTy())
2428 StoreVal = Builder.CreatePtrToInt(V: StoreVal, DestTy: Int32Ty);
2429 else {
2430 llvm::Type *IntTy = llvm::IntegerType::get(
2431 C&: getLLVMContext(),
2432 NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: StoreVal->getType()));
2433 StoreVal = Builder.CreateBitCast(V: StoreVal, DestTy: IntTy);
2434 StoreVal = Builder.CreateZExtOrBitCast(V: StoreVal, DestTy: Int32Ty);
2435 }
2436
2437 Function *F = CGM.getIntrinsic(
2438 IID: BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
2439 : Intrinsic::arm_strex,
2440 Tys: StoreAddr->getType());
2441
2442 CallInst *CI = Builder.CreateCall(Callee: F, Args: {StoreVal, StoreAddr}, Name: "strex");
2443 CI->addParamAttr(
2444 ArgNo: 1, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: StoreTy));
2445 return CI;
2446 }
2447
2448 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
2449 Function *F = CGM.getIntrinsic(IID: Intrinsic::arm_clrex);
2450 return Builder.CreateCall(Callee: F);
2451 }
2452
2453 // CRC32
2454 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
2455 switch (BuiltinID) {
2456 case clang::ARM::BI__builtin_arm_crc32b:
2457 CRCIntrinsicID = Intrinsic::arm_crc32b; break;
2458 case clang::ARM::BI__builtin_arm_crc32cb:
2459 CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
2460 case clang::ARM::BI__builtin_arm_crc32h:
2461 CRCIntrinsicID = Intrinsic::arm_crc32h; break;
2462 case clang::ARM::BI__builtin_arm_crc32ch:
2463 CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
2464 case clang::ARM::BI__builtin_arm_crc32w:
2465 case clang::ARM::BI__builtin_arm_crc32d:
2466 CRCIntrinsicID = Intrinsic::arm_crc32w; break;
2467 case clang::ARM::BI__builtin_arm_crc32cw:
2468 case clang::ARM::BI__builtin_arm_crc32cd:
2469 CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
2470 }
2471
2472 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
2473 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
2474 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
2475
2476 // crc32{c,}d intrinsics are implemented as two calls to crc32{c,}w
2477 // intrinsics, hence we need different codegen for these cases.
2478 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
2479 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
2480 Value *C1 = llvm::ConstantInt::get(Ty: Int64Ty, V: 32);
2481 Value *Arg1a = Builder.CreateTruncOrBitCast(V: Arg1, DestTy: Int32Ty);
2482 Value *Arg1b = Builder.CreateLShr(LHS: Arg1, RHS: C1);
2483 Arg1b = Builder.CreateTruncOrBitCast(V: Arg1b, DestTy: Int32Ty);
2484
2485 Function *F = CGM.getIntrinsic(IID: CRCIntrinsicID);
2486 Value *Res = Builder.CreateCall(Callee: F, Args: {Arg0, Arg1a});
2487 return Builder.CreateCall(Callee: F, Args: {Res, Arg1b});
2488 } else {
2489 Arg1 = Builder.CreateZExtOrBitCast(V: Arg1, DestTy: Int32Ty);
2490
2491 Function *F = CGM.getIntrinsic(IID: CRCIntrinsicID);
2492 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1});
2493 }
2494 }
2495
2496 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2497 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2498 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2499 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
2500 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
2501 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
2502
2503 SpecialRegisterAccessKind AccessKind = Write;
2504 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2505 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2506 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
2507 AccessKind = VolatileRead;
2508
2509 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2510 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
2511
2512 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2513 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
2514
2515 llvm::Type *ValueType;
2516 llvm::Type *RegisterType;
2517 if (IsPointerBuiltin) {
2518 ValueType = VoidPtrTy;
2519 RegisterType = Int32Ty;
2520 } else if (Is64Bit) {
2521 ValueType = RegisterType = Int64Ty;
2522 } else {
2523 ValueType = RegisterType = Int32Ty;
2524 }
2525
2526 return EmitSpecialRegisterBuiltin(CGF&: *this, E, RegisterType, ValueType,
2527 AccessKind);
2528 }
2529
2530 if (BuiltinID == ARM::BI__builtin_sponentry) {
2531 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: AllocaInt8PtrTy);
2532 return Builder.CreateCall(Callee: F);
2533 }
2534
2535 // Handle MSVC intrinsics before argument evaluation to prevent double
2536 // evaluation.
2537 if (std::optional<MSVCIntrin> MsvcIntId = translateArmToMsvcIntrin(BuiltinID))
2538 return EmitMSVCBuiltinExpr(BuiltinID: *MsvcIntId, E);
2539
2540 // Deal with MVE builtins
2541 if (Value *Result = EmitARMMVEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
2542 return Result;
2543 // Handle CDE builtins
2544 if (Value *Result = EmitARMCDEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
2545 return Result;
2546
2547 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
2548 auto It = llvm::find_if(Range: NEONEquivalentIntrinsicMap, P: [BuiltinID](auto &P) {
2549 return P.first == BuiltinID;
2550 });
2551 if (It != end(arr: NEONEquivalentIntrinsicMap))
2552 BuiltinID = It->second;
2553
2554 // Find out if any arguments are required to be integer constant
2555 // expressions.
2556 unsigned ICEArguments = 0;
2557 ASTContext::GetBuiltinTypeError Error;
2558 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
2559 assert(Error == ASTContext::GE_None && "Should not codegen an error");
2560
2561 auto getAlignmentValue32 = [&](Address addr) -> Value* {
2562 return Builder.getInt32(C: addr.getAlignment().getQuantity());
2563 };
2564
2565 Address PtrOp0 = Address::invalid();
2566 Address PtrOp1 = Address::invalid();
2567 SmallVector<Value*, 4> Ops;
2568 bool HasExtraArg = CodeGenUtils::hasExtraNeonArgument(BuiltinID);
2569 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
2570 for (unsigned i = 0, e = NumArgs; i != e; i++) {
2571 if (i == 0) {
2572 switch (BuiltinID) {
2573 case NEON::BI__builtin_neon_vld1_v:
2574 case NEON::BI__builtin_neon_vld1q_v:
2575 case NEON::BI__builtin_neon_vld1q_lane_v:
2576 case NEON::BI__builtin_neon_vld1_lane_v:
2577 case NEON::BI__builtin_neon_vld1_dup_v:
2578 case NEON::BI__builtin_neon_vld1q_dup_v:
2579 case NEON::BI__builtin_neon_vst1_v:
2580 case NEON::BI__builtin_neon_vst1q_v:
2581 case NEON::BI__builtin_neon_vst1q_lane_v:
2582 case NEON::BI__builtin_neon_vst1_lane_v:
2583 case NEON::BI__builtin_neon_vst2_v:
2584 case NEON::BI__builtin_neon_vst2q_v:
2585 case NEON::BI__builtin_neon_vst2_lane_v:
2586 case NEON::BI__builtin_neon_vst2q_lane_v:
2587 case NEON::BI__builtin_neon_vst3_v:
2588 case NEON::BI__builtin_neon_vst3q_v:
2589 case NEON::BI__builtin_neon_vst3_lane_v:
2590 case NEON::BI__builtin_neon_vst3q_lane_v:
2591 case NEON::BI__builtin_neon_vst4_v:
2592 case NEON::BI__builtin_neon_vst4q_v:
2593 case NEON::BI__builtin_neon_vst4_lane_v:
2594 case NEON::BI__builtin_neon_vst4q_lane_v:
2595 // Get the alignment for the argument in addition to the value;
2596 // we'll use it later.
2597 PtrOp0 = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
2598 Ops.push_back(Elt: PtrOp0.emitRawPointer(CGF&: *this));
2599 continue;
2600 }
2601 }
2602 if (i == 1) {
2603 switch (BuiltinID) {
2604 case NEON::BI__builtin_neon_vld2_v:
2605 case NEON::BI__builtin_neon_vld2q_v:
2606 case NEON::BI__builtin_neon_vld3_v:
2607 case NEON::BI__builtin_neon_vld3q_v:
2608 case NEON::BI__builtin_neon_vld4_v:
2609 case NEON::BI__builtin_neon_vld4q_v:
2610 case NEON::BI__builtin_neon_vld2_lane_v:
2611 case NEON::BI__builtin_neon_vld2q_lane_v:
2612 case NEON::BI__builtin_neon_vld3_lane_v:
2613 case NEON::BI__builtin_neon_vld3q_lane_v:
2614 case NEON::BI__builtin_neon_vld4_lane_v:
2615 case NEON::BI__builtin_neon_vld4q_lane_v:
2616 case NEON::BI__builtin_neon_vld2_dup_v:
2617 case NEON::BI__builtin_neon_vld2q_dup_v:
2618 case NEON::BI__builtin_neon_vld3_dup_v:
2619 case NEON::BI__builtin_neon_vld3q_dup_v:
2620 case NEON::BI__builtin_neon_vld4_dup_v:
2621 case NEON::BI__builtin_neon_vld4q_dup_v:
2622 // Get the alignment for the argument in addition to the value;
2623 // we'll use it later.
2624 PtrOp1 = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
2625 Ops.push_back(Elt: PtrOp1.emitRawPointer(CGF&: *this));
2626 continue;
2627 }
2628 }
2629
2630 Ops.push_back(Elt: EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E));
2631 }
2632
2633 switch (BuiltinID) {
2634 default: break;
2635
2636 case NEON::BI__builtin_neon_vget_lane_i8:
2637 case NEON::BI__builtin_neon_vget_lane_i16:
2638 case NEON::BI__builtin_neon_vget_lane_i32:
2639 case NEON::BI__builtin_neon_vget_lane_i64:
2640 case NEON::BI__builtin_neon_vget_lane_bf16:
2641 case NEON::BI__builtin_neon_vget_lane_f32:
2642 case NEON::BI__builtin_neon_vgetq_lane_i8:
2643 case NEON::BI__builtin_neon_vgetq_lane_i16:
2644 case NEON::BI__builtin_neon_vgetq_lane_i32:
2645 case NEON::BI__builtin_neon_vgetq_lane_i64:
2646 case NEON::BI__builtin_neon_vgetq_lane_bf16:
2647 case NEON::BI__builtin_neon_vgetq_lane_f32:
2648 case NEON::BI__builtin_neon_vduph_lane_bf16:
2649 case NEON::BI__builtin_neon_vduph_laneq_bf16:
2650 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
2651
2652 case NEON::BI__builtin_neon_vrndns_f32: {
2653 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2654 llvm::Type *Tys[] = {Arg->getType()};
2655 Function *F = CGM.getIntrinsic(IID: Intrinsic::roundeven, Tys);
2656 return Builder.CreateCall(Callee: F, Args: {Arg}, Name: "vrndn"); }
2657
2658 case NEON::BI__builtin_neon_vset_lane_i8:
2659 case NEON::BI__builtin_neon_vset_lane_i16:
2660 case NEON::BI__builtin_neon_vset_lane_i32:
2661 case NEON::BI__builtin_neon_vset_lane_i64:
2662 case NEON::BI__builtin_neon_vset_lane_bf16:
2663 case NEON::BI__builtin_neon_vset_lane_f32:
2664 case NEON::BI__builtin_neon_vsetq_lane_i8:
2665 case NEON::BI__builtin_neon_vsetq_lane_i16:
2666 case NEON::BI__builtin_neon_vsetq_lane_i32:
2667 case NEON::BI__builtin_neon_vsetq_lane_i64:
2668 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2669 case NEON::BI__builtin_neon_vsetq_lane_f32:
2670 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
2671
2672 case NEON::BI__builtin_neon_vsha1h_u32:
2673 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1h), Ops,
2674 name: "vsha1h");
2675 case NEON::BI__builtin_neon_vsha1cq_u32:
2676 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1c), Ops,
2677 name: "vsha1h");
2678 case NEON::BI__builtin_neon_vsha1pq_u32:
2679 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1p), Ops,
2680 name: "vsha1h");
2681 case NEON::BI__builtin_neon_vsha1mq_u32:
2682 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_sha1m), Ops,
2683 name: "vsha1h");
2684
2685 case NEON::BI__builtin_neon_vcvth_bf16_f32:
2686 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vcvtbfp2bf), Ops,
2687 name: "vcvtbfp2bf");
2688 case NEON::BI__builtin_neon_vcvt_f16_f32:
2689 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vcvtfp2hf), Ops,
2690 name: "vcvtfp2hf");
2691 case NEON::BI__builtin_neon_vcvt_f32_f16:
2692 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vcvthf2fp), Ops,
2693 name: "vcvthf2fp");
2694
2695 // The ARM _MoveToCoprocessor builtins put the input register value as
2696 // the first argument, but the LLVM intrinsic expects it as the third one.
2697 case clang::ARM::BI_MoveToCoprocessor:
2698 case clang::ARM::BI_MoveToCoprocessor2: {
2699 Function *F = CGM.getIntrinsic(IID: BuiltinID == clang::ARM::BI_MoveToCoprocessor
2700 ? Intrinsic::arm_mcr
2701 : Intrinsic::arm_mcr2);
2702 return Builder.CreateCall(Callee: F, Args: {Ops[1], Ops[2], Ops[0],
2703 Ops[3], Ops[4], Ops[5]});
2704 }
2705 }
2706
2707 // Get the last argument, which specifies the vector type.
2708 assert(HasExtraArg);
2709 const Expr *Arg = E->getArg(Arg: E->getNumArgs()-1);
2710 std::optional<llvm::APSInt> Result =
2711 Arg->getIntegerConstantExpr(Ctx: getContext());
2712 if (!Result)
2713 return nullptr;
2714
2715 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
2716 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
2717 // Determine the overloaded type of this builtin.
2718 llvm::Type *Ty;
2719 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
2720 Ty = FloatTy;
2721 else
2722 Ty = DoubleTy;
2723
2724 // Determine whether this is an unsigned conversion or not.
2725 bool usgn = Result->getZExtValue() == 1;
2726 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
2727
2728 // Call the appropriate intrinsic.
2729 Function *F = CGM.getIntrinsic(IID: Int, Tys: Ty);
2730 return Builder.CreateCall(Callee: F, Args: Ops, Name: "vcvtr");
2731 }
2732
2733 // Determine the type of this overloaded NEON intrinsic.
2734 NeonTypeFlags Type = Result->getZExtValue();
2735 bool usgn = Type.isUnsigned();
2736 bool rightShift = false;
2737
2738 llvm::FixedVectorType *VTy =
2739 GetNeonType(CGF: this, TypeFlags: Type, HasFastHalfType: getTarget().hasFastHalfType(), V1Ty: false,
2740 AllowBFloatArgsAndRet: getTarget().hasBFloat16Type());
2741 llvm::Type *Ty = VTy;
2742 if (!Ty)
2743 return nullptr;
2744
2745 // Many NEON builtins have identical semantics and uses in ARM and
2746 // AArch64. Emit these in a single function.
2747 auto IntrinsicMap = ArrayRef(ARMSIMDIntrinsicMap);
2748 const ARMNeonVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap(
2749 IntrinsicMap, BuiltinID, MapProvenSorted&: NEONSIMDIntrinsicsProvenSorted);
2750 if (Builtin)
2751 return EmitCommonNeonBuiltinExpr(
2752 BuiltinID: Builtin->BuiltinID, LLVMIntrinsic: Builtin->LLVMIntrinsic, AltLLVMIntrinsic: Builtin->AltLLVMIntrinsic,
2753 NameHint: Builtin->NameHint, Modifier: Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
2754
2755 unsigned Int;
2756 switch (BuiltinID) {
2757 default: return nullptr;
2758 case NEON::BI__builtin_neon_vld1q_lane_v:
2759 // Handle 64-bit integer elements as a special case. Use shuffles of
2760 // one-element vectors to avoid poor code for i64 in the backend.
2761 if (VTy->getElementType()->isIntegerTy(BitWidth: 64)) {
2762 // Extract the other lane.
2763 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2764 int Lane = cast<ConstantInt>(Val: Ops[2])->getZExtValue();
2765 Value *SV = llvm::ConstantVector::get(V: ConstantInt::get(Ty: Int32Ty, V: 1-Lane));
2766 Ops[1] = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[1], Mask: SV);
2767 // Load the value as a one-element vector.
2768 Ty = llvm::FixedVectorType::get(ElementType: VTy->getElementType(), NumElts: 1);
2769 llvm::Type *Tys[] = {Ty, Int8PtrTy};
2770 Function *F = CGM.getIntrinsic(IID: Intrinsic::arm_neon_vld1, Tys);
2771 Value *Align = getAlignmentValue32(PtrOp0);
2772 Value *Ld = Builder.CreateCall(Callee: F, Args: {Ops[0], Align});
2773 // Combine them.
2774 int Indices[] = {1 - Lane, Lane};
2775 return Builder.CreateShuffleVector(V1: Ops[1], V2: Ld, Mask: Indices, Name: "vld1q_lane");
2776 }
2777 [[fallthrough]];
2778 case NEON::BI__builtin_neon_vld1_lane_v: {
2779 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2780 PtrOp0 = PtrOp0.withElementType(ElemTy: VTy->getElementType());
2781 Value *Ld = Builder.CreateLoad(Addr: PtrOp0);
2782 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ld, Idx: Ops[2], Name: "vld1_lane");
2783 }
2784 case NEON::BI__builtin_neon_vqrshrn_n_v:
2785 Int =
2786 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
2787 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqrshrn_n",
2788 shift: 1, rightshift: true);
2789 case NEON::BI__builtin_neon_vqrshrun_n_v:
2790 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vqrshiftnsu, Tys: Ty),
2791 Ops, name: "vqrshrun_n", shift: 1, rightshift: true);
2792 case NEON::BI__builtin_neon_vqshrn_n_v:
2793 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
2794 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshrn_n",
2795 shift: 1, rightshift: true);
2796 case NEON::BI__builtin_neon_vqshrun_n_v:
2797 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vqshiftnsu, Tys: Ty),
2798 Ops, name: "vqshrun_n", shift: 1, rightshift: true);
2799 case NEON::BI__builtin_neon_vrecpe_v:
2800 case NEON::BI__builtin_neon_vrecpeq_v:
2801 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vrecpe, Tys: Ty),
2802 Ops, name: "vrecpe");
2803 case NEON::BI__builtin_neon_vrshrn_n_v:
2804 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vrshiftn, Tys: Ty),
2805 Ops, name: "vrshrn_n", shift: 1, rightshift: true);
2806 case NEON::BI__builtin_neon_vrsra_n_v:
2807 case NEON::BI__builtin_neon_vrsraq_n_v:
2808 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
2809 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2810 Ops[2] = EmitNeonShiftVector(V: Ops[2], Ty, neg: true);
2811 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
2812 Ops[1] = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Int, Tys: Ty), Args: {Ops[1], Ops[2]});
2813 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1], Name: "vrsra_n");
2814 case NEON::BI__builtin_neon_vsri_n_v:
2815 case NEON::BI__builtin_neon_vsriq_n_v:
2816 rightShift = true;
2817 [[fallthrough]];
2818 case NEON::BI__builtin_neon_vsli_n_v:
2819 case NEON::BI__builtin_neon_vsliq_n_v:
2820 Ops[2] = EmitNeonShiftVector(V: Ops[2], Ty, neg: rightShift);
2821 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vshiftins, Tys: Ty),
2822 Ops, name: "vsli_n");
2823 case NEON::BI__builtin_neon_vsra_n_v:
2824 case NEON::BI__builtin_neon_vsraq_n_v:
2825 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
2826 Ops[1] = EmitNeonRShiftImm(Vec: Ops[1], Shift: Ops[2], Ty, usgn, name: "vsra_n");
2827 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
2828 case NEON::BI__builtin_neon_vst1q_lane_v:
2829 // Handle 64-bit integer elements as a special case. Use a shuffle to get
2830 // a one-element vector and avoid poor code for i64 in the backend.
2831 if (VTy->getElementType()->isIntegerTy(BitWidth: 64)) {
2832 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2833 Value *SV = llvm::ConstantVector::get(V: cast<llvm::Constant>(Val: Ops[2]));
2834 Ops[1] = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[1], Mask: SV);
2835 Ops[2] = getAlignmentValue32(PtrOp0);
2836 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
2837 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vst1,
2838 Tys), Args: Ops);
2839 }
2840 [[fallthrough]];
2841 case NEON::BI__builtin_neon_vst1_lane_v: {
2842 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
2843 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2]);
2844 return Builder.CreateStore(Val: Ops[1],
2845 Addr: PtrOp0.withElementType(ElemTy: Ops[1]->getType()));
2846 }
2847 case NEON::BI__builtin_neon_vtbl1_v:
2848 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl1),
2849 Ops, name: "vtbl1");
2850 case NEON::BI__builtin_neon_vtbl2_v:
2851 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl2),
2852 Ops, name: "vtbl2");
2853 case NEON::BI__builtin_neon_vtbl3_v:
2854 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl3),
2855 Ops, name: "vtbl3");
2856 case NEON::BI__builtin_neon_vtbl4_v:
2857 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbl4),
2858 Ops, name: "vtbl4");
2859 case NEON::BI__builtin_neon_vtbx1_v:
2860 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx1),
2861 Ops, name: "vtbx1");
2862 case NEON::BI__builtin_neon_vtbx2_v:
2863 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx2),
2864 Ops, name: "vtbx2");
2865 case NEON::BI__builtin_neon_vtbx3_v:
2866 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx3),
2867 Ops, name: "vtbx3");
2868 case NEON::BI__builtin_neon_vtbx4_v:
2869 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::arm_neon_vtbx4),
2870 Ops, name: "vtbx4");
2871 }
2872}
2873
2874template<typename Integer>
2875static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context) {
2876 return E->getIntegerConstantExpr(Ctx: Context)->getExtValue();
2877}
2878
2879static llvm::Value *SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V,
2880 llvm::Type *T, bool Unsigned) {
2881 // Helper function called by Tablegen-constructed ARM MVE builtin codegen,
2882 // which finds it convenient to specify signed/unsigned as a boolean flag.
2883 return Unsigned ? Builder.CreateZExt(V, DestTy: T) : Builder.CreateSExt(V, DestTy: T);
2884}
2885
2886static llvm::Value *MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V,
2887 uint32_t Shift, bool Unsigned) {
2888 // MVE helper function for integer shift right. This must handle signed vs
2889 // unsigned, and also deal specially with the case where the shift count is
2890 // equal to the lane size. In LLVM IR, an LShr with that parameter would be
2891 // undefined behavior, but in MVE it's legal, so we must convert it to code
2892 // that is not undefined in IR.
2893 unsigned LaneBits = cast<llvm::VectorType>(Val: V->getType())
2894 ->getElementType()
2895 ->getPrimitiveSizeInBits();
2896 if (Shift == LaneBits) {
2897 // An unsigned shift of the full lane size always generates zero, so we can
2898 // simply emit a zero vector. A signed shift of the full lane size does the
2899 // same thing as shifting by one bit fewer.
2900 if (Unsigned)
2901 return llvm::Constant::getNullValue(Ty: V->getType());
2902 else
2903 --Shift;
2904 }
2905 return Unsigned ? Builder.CreateLShr(LHS: V, RHS: Shift) : Builder.CreateAShr(LHS: V, RHS: Shift);
2906}
2907
2908static llvm::Value *ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V) {
2909 // MVE-specific helper function for a vector splat, which infers the element
2910 // count of the output vector by knowing that MVE vectors are all 128 bits
2911 // wide.
2912 unsigned Elements = 128 / V->getType()->getPrimitiveSizeInBits();
2913 return Builder.CreateVectorSplat(NumElts: Elements, V);
2914}
2915
2916static llvm::Value *ARMMVEVectorReinterpret(CGBuilderTy &Builder,
2917 CodeGenFunction *CGF,
2918 llvm::Value *V,
2919 llvm::Type *DestType) {
2920 // Convert one MVE vector type into another by reinterpreting its in-register
2921 // format.
2922 //
2923 // Little-endian, this is identical to a bitcast (which reinterprets the
2924 // memory format). But big-endian, they're not necessarily the same, because
2925 // the register and memory formats map to each other differently depending on
2926 // the lane size.
2927 //
2928 // We generate a bitcast whenever we can (if we're little-endian, or if the
2929 // lane sizes are the same anyway). Otherwise we fall back to an IR intrinsic
2930 // that performs the different kind of reinterpretation.
2931 if (CGF->getTarget().isBigEndian() &&
2932 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
2933 return Builder.CreateCall(
2934 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vreinterpretq,
2935 Tys: {DestType, V->getType()}),
2936 Args: V);
2937 } else {
2938 return Builder.CreateBitCast(V, DestTy: DestType);
2939 }
2940}
2941
2942static llvm::Value *VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd) {
2943 // Make a shufflevector that extracts every other element of a vector (evens
2944 // or odds, as desired).
2945 SmallVector<int, 16> Indices;
2946 unsigned InputElements =
2947 cast<llvm::FixedVectorType>(Val: V->getType())->getNumElements();
2948 for (unsigned i = 0; i < InputElements; i += 2)
2949 Indices.push_back(Elt: i + Odd);
2950 return Builder.CreateShuffleVector(V, Mask: Indices);
2951}
2952
2953static llvm::Value *VectorZip(CGBuilderTy &Builder, llvm::Value *V0,
2954 llvm::Value *V1) {
2955 // Make a shufflevector that interleaves two vectors element by element.
2956 assert(V0->getType() == V1->getType() && "Can't zip different vector types");
2957 SmallVector<int, 16> Indices;
2958 unsigned InputElements =
2959 cast<llvm::FixedVectorType>(Val: V0->getType())->getNumElements();
2960 for (unsigned i = 0; i < InputElements; i++) {
2961 Indices.push_back(Elt: i);
2962 Indices.push_back(Elt: i + InputElements);
2963 }
2964 return Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: Indices);
2965}
2966
2967template<unsigned HighBit, unsigned OtherBits>
2968static llvm::Value *ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT) {
2969 // MVE-specific helper function to make a vector splat of a constant such as
2970 // UINT_MAX or INT_MIN, in which all bits below the highest one are equal.
2971 llvm::Type *T = cast<llvm::VectorType>(Val: VT)->getElementType();
2972 unsigned LaneBits = T->getPrimitiveSizeInBits();
2973 uint32_t Value = HighBit << (LaneBits - 1);
2974 if (OtherBits)
2975 Value |= (1UL << (LaneBits - 1)) - 1;
2976 llvm::Value *Lane = llvm::ConstantInt::get(Ty: T, V: Value);
2977 return ARMMVEVectorSplat(Builder, V: Lane);
2978}
2979
2980static llvm::Value *ARMMVEVectorElementReverse(CGBuilderTy &Builder,
2981 llvm::Value *V,
2982 unsigned ReverseWidth) {
2983 // MVE-specific helper function which reverses the elements of a
2984 // vector within every (ReverseWidth)-bit collection of lanes.
2985 SmallVector<int, 16> Indices;
2986 unsigned LaneSize = V->getType()->getScalarSizeInBits();
2987 unsigned Elements = 128 / LaneSize;
2988 unsigned Mask = ReverseWidth / LaneSize - 1;
2989 for (unsigned i = 0; i < Elements; i++)
2990 Indices.push_back(Elt: i ^ Mask);
2991 return Builder.CreateShuffleVector(V, Mask: Indices);
2992}
2993
2994static llvm::Value *ARMMVECreateSIToFP(CGBuilderTy &Builder,
2995 CodeGenFunction *CGF, llvm::Value *V,
2996 llvm::Type *Ty) {
2997 return Builder.CreateCall(
2998 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_fp_int, Tys: {Ty, V->getType()}),
2999 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)});
3000}
3001
3002static llvm::Value *ARMMVECreateUIToFP(CGBuilderTy &Builder,
3003 CodeGenFunction *CGF, llvm::Value *V,
3004 llvm::Type *Ty) {
3005 return Builder.CreateCall(
3006 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_fp_int, Tys: {Ty, V->getType()}),
3007 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1)});
3008}
3009
3010static llvm::Value *ARMMVECreateFPToSI(CGBuilderTy &Builder,
3011 CodeGenFunction *CGF, llvm::Value *V,
3012 llvm::Type *Ty) {
3013 return Builder.CreateCall(
3014 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_int_fp, Tys: {Ty, V->getType()}),
3015 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)});
3016}
3017
3018static llvm::Value *ARMMVECreateFPToUI(CGBuilderTy &Builder,
3019 CodeGenFunction *CGF, llvm::Value *V,
3020 llvm::Type *Ty) {
3021 return Builder.CreateCall(
3022 Callee: CGF->CGM.getIntrinsic(IID: Intrinsic::arm_mve_vcvt_int_fp, Tys: {Ty, V->getType()}),
3023 Args: {V, llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1)});
3024}
3025
3026Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID,
3027 const CallExpr *E,
3028 ReturnValueSlot ReturnValue,
3029 llvm::Triple::ArchType Arch) {
3030 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
3031 Intrinsic::ID IRIntr;
3032 unsigned NumVectors;
3033
3034 // Code autogenerated by Tablegen will handle all the simple builtins.
3035 switch (BuiltinID) {
3036 #include "clang/Basic/arm_mve_builtin_cg.inc"
3037
3038 // If we didn't match an MVE builtin id at all, go back to the
3039 // main EmitARMBuiltinExpr.
3040 default:
3041 return nullptr;
3042 }
3043
3044 // Anything that breaks from that switch is an MVE builtin that
3045 // needs handwritten code to generate.
3046
3047 switch (CustomCodeGenType) {
3048
3049 case CustomCodeGen::VLD24: {
3050 llvm::SmallVector<Value *, 4> Ops;
3051 llvm::SmallVector<llvm::Type *, 4> Tys;
3052
3053 auto MvecCType = E->getType();
3054 auto MvecLType = ConvertType(T: MvecCType);
3055 assert(MvecLType->isStructTy() &&
3056 "Return type for vld[24]q should be a struct");
3057 assert(MvecLType->getStructNumElements() == 1 &&
3058 "Return-type struct for vld[24]q should have one element");
3059 auto MvecLTypeInner = MvecLType->getStructElementType(N: 0);
3060 assert(MvecLTypeInner->isArrayTy() &&
3061 "Return-type struct for vld[24]q should contain an array");
3062 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3063 "Array member of return-type struct vld[24]q has wrong length");
3064 auto VecLType = MvecLTypeInner->getArrayElementType();
3065
3066 Tys.push_back(Elt: VecLType);
3067
3068 auto Addr = E->getArg(Arg: 0);
3069 Ops.push_back(Elt: EmitScalarExpr(E: Addr));
3070 Tys.push_back(Elt: ConvertType(T: Addr->getType()));
3071
3072 Function *F = CGM.getIntrinsic(IID: IRIntr, Tys: ArrayRef(Tys));
3073 Value *LoadResult = Builder.CreateCall(Callee: F, Args: Ops);
3074 Value *MvecOut = PoisonValue::get(T: MvecLType);
3075 for (unsigned i = 0; i < NumVectors; ++i) {
3076 Value *Vec = Builder.CreateExtractValue(Agg: LoadResult, Idxs: i);
3077 MvecOut = Builder.CreateInsertValue(Agg: MvecOut, Val: Vec, Idxs: {0, i});
3078 }
3079
3080 if (ReturnValue.isNull())
3081 return MvecOut;
3082 else
3083 return Builder.CreateStore(Val: MvecOut, Addr: ReturnValue.getAddress());
3084 }
3085
3086 case CustomCodeGen::VST24: {
3087 llvm::SmallVector<Value *, 4> Ops;
3088 llvm::SmallVector<llvm::Type *, 4> Tys;
3089
3090 auto Addr = E->getArg(Arg: 0);
3091 Ops.push_back(Elt: EmitScalarExpr(E: Addr));
3092 Tys.push_back(Elt: ConvertType(T: Addr->getType()));
3093
3094 auto MvecCType = E->getArg(Arg: 1)->getType();
3095 auto MvecLType = ConvertType(T: MvecCType);
3096 assert(MvecLType->isStructTy() && "Data type for vst2q should be a struct");
3097 assert(MvecLType->getStructNumElements() == 1 &&
3098 "Data-type struct for vst2q should have one element");
3099 auto MvecLTypeInner = MvecLType->getStructElementType(N: 0);
3100 assert(MvecLTypeInner->isArrayTy() &&
3101 "Data-type struct for vst2q should contain an array");
3102 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3103 "Array member of return-type struct vld[24]q has wrong length");
3104 auto VecLType = MvecLTypeInner->getArrayElementType();
3105
3106 Tys.push_back(Elt: VecLType);
3107
3108 AggValueSlot MvecSlot = CreateAggTemp(T: MvecCType);
3109 EmitAggExpr(E: E->getArg(Arg: 1), AS: MvecSlot);
3110 auto Mvec = Builder.CreateLoad(Addr: MvecSlot.getAddress());
3111 for (unsigned i = 0; i < NumVectors; i++)
3112 Ops.push_back(Elt: Builder.CreateExtractValue(Agg: Mvec, Idxs: {0, i}));
3113
3114 Function *F = CGM.getIntrinsic(IID: IRIntr, Tys: ArrayRef(Tys));
3115 Value *ToReturn = nullptr;
3116 for (unsigned i = 0; i < NumVectors; i++) {
3117 Ops.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: i));
3118 ToReturn = Builder.CreateCall(Callee: F, Args: Ops);
3119 Ops.pop_back();
3120 }
3121 return ToReturn;
3122 }
3123 }
3124 llvm_unreachable("unknown custom codegen type.");
3125}
3126
3127Value *CodeGenFunction::EmitARMCDEBuiltinExpr(unsigned BuiltinID,
3128 const CallExpr *E,
3129 ReturnValueSlot ReturnValue,
3130 llvm::Triple::ArchType Arch) {
3131 switch (BuiltinID) {
3132 default:
3133 return nullptr;
3134#include "clang/Basic/arm_cde_builtin_cg.inc"
3135 }
3136}
3137
3138static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
3139 const CallExpr *E,
3140 SmallVectorImpl<Value *> &Ops,
3141 llvm::Triple::ArchType Arch) {
3142 unsigned int Int = 0;
3143 const char *s = nullptr;
3144
3145 switch (BuiltinID) {
3146 default:
3147 return nullptr;
3148 case NEON::BI__builtin_neon_vtbl1_v:
3149 case NEON::BI__builtin_neon_vqtbl1_v:
3150 case NEON::BI__builtin_neon_vqtbl1q_v:
3151 case NEON::BI__builtin_neon_vtbl2_v:
3152 case NEON::BI__builtin_neon_vqtbl2_v:
3153 case NEON::BI__builtin_neon_vqtbl2q_v:
3154 case NEON::BI__builtin_neon_vtbl3_v:
3155 case NEON::BI__builtin_neon_vqtbl3_v:
3156 case NEON::BI__builtin_neon_vqtbl3q_v:
3157 case NEON::BI__builtin_neon_vtbl4_v:
3158 case NEON::BI__builtin_neon_vqtbl4_v:
3159 case NEON::BI__builtin_neon_vqtbl4q_v:
3160 break;
3161 case NEON::BI__builtin_neon_vtbx1_v:
3162 case NEON::BI__builtin_neon_vqtbx1_v:
3163 case NEON::BI__builtin_neon_vqtbx1q_v:
3164 case NEON::BI__builtin_neon_vtbx2_v:
3165 case NEON::BI__builtin_neon_vqtbx2_v:
3166 case NEON::BI__builtin_neon_vqtbx2q_v:
3167 case NEON::BI__builtin_neon_vtbx3_v:
3168 case NEON::BI__builtin_neon_vqtbx3_v:
3169 case NEON::BI__builtin_neon_vqtbx3q_v:
3170 case NEON::BI__builtin_neon_vtbx4_v:
3171 case NEON::BI__builtin_neon_vqtbx4_v:
3172 case NEON::BI__builtin_neon_vqtbx4q_v:
3173 break;
3174 }
3175
3176 assert(E->getNumArgs() >= 3);
3177
3178 // Get the last argument, which specifies the vector type.
3179 const Expr *Arg = E->getArg(Arg: E->getNumArgs() - 1);
3180 std::optional<llvm::APSInt> Result =
3181 Arg->getIntegerConstantExpr(Ctx: CGF.getContext());
3182 if (!Result)
3183 return nullptr;
3184
3185 // Determine the type of this overloaded NEON intrinsic.
3186 NeonTypeFlags Type = Result->getZExtValue();
3187 llvm::FixedVectorType *Ty = GetNeonType(CGF: &CGF, TypeFlags: Type);
3188 if (!Ty)
3189 return nullptr;
3190
3191 CodeGen::CGBuilderTy &Builder = CGF.Builder;
3192
3193 // AArch64 scalar builtins are not overloaded, they do not have an extra
3194 // argument that specifies the vector type, need to handle each case.
3195 switch (BuiltinID) {
3196 case NEON::BI__builtin_neon_vtbl1_v: {
3197 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 1), ExtOp: nullptr, IndexOp: Ops[1],
3198 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl1, Name: "vtbl1");
3199 }
3200 case NEON::BI__builtin_neon_vtbl2_v: {
3201 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 2), ExtOp: nullptr, IndexOp: Ops[2],
3202 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl1, Name: "vtbl1");
3203 }
3204 case NEON::BI__builtin_neon_vtbl3_v: {
3205 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 3), ExtOp: nullptr, IndexOp: Ops[3],
3206 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl2, Name: "vtbl2");
3207 }
3208 case NEON::BI__builtin_neon_vtbl4_v: {
3209 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 0, M: 4), ExtOp: nullptr, IndexOp: Ops[4],
3210 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbl2, Name: "vtbl2");
3211 }
3212 case NEON::BI__builtin_neon_vtbx1_v: {
3213 Value *TblRes =
3214 packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 1), ExtOp: nullptr, IndexOp: Ops[2], ResTy: Ty,
3215 IntID: Intrinsic::aarch64_neon_tbl1, Name: "vtbl1");
3216
3217 llvm::Constant *EightV = ConstantInt::get(Ty, V: 8);
3218 Value *CmpRes = Builder.CreateICmp(P: ICmpInst::ICMP_UGE, LHS: Ops[2], RHS: EightV);
3219 CmpRes = Builder.CreateSExt(V: CmpRes, DestTy: Ty);
3220
3221 Value *EltsFromInput = Builder.CreateAnd(LHS: CmpRes, RHS: Ops[0]);
3222 Value *EltsFromTbl = Builder.CreateAnd(LHS: Builder.CreateNot(V: CmpRes), RHS: TblRes);
3223 return Builder.CreateOr(LHS: EltsFromInput, RHS: EltsFromTbl, Name: "vtbx");
3224 }
3225 case NEON::BI__builtin_neon_vtbx2_v: {
3226 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 2), ExtOp: Ops[0], IndexOp: Ops[3],
3227 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbx1, Name: "vtbx1");
3228 }
3229 case NEON::BI__builtin_neon_vtbx3_v: {
3230 Value *TblRes =
3231 packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 3), ExtOp: nullptr, IndexOp: Ops[4], ResTy: Ty,
3232 IntID: Intrinsic::aarch64_neon_tbl2, Name: "vtbl2");
3233
3234 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, V: 24);
3235 Value *CmpRes = Builder.CreateICmp(P: ICmpInst::ICMP_UGE, LHS: Ops[4],
3236 RHS: TwentyFourV);
3237 CmpRes = Builder.CreateSExt(V: CmpRes, DestTy: Ty);
3238
3239 Value *EltsFromInput = Builder.CreateAnd(LHS: CmpRes, RHS: Ops[0]);
3240 Value *EltsFromTbl = Builder.CreateAnd(LHS: Builder.CreateNot(V: CmpRes), RHS: TblRes);
3241 return Builder.CreateOr(LHS: EltsFromInput, RHS: EltsFromTbl, Name: "vtbx");
3242 }
3243 case NEON::BI__builtin_neon_vtbx4_v: {
3244 return packTBLDVectorList(CGF, Ops: ArrayRef(Ops).slice(N: 1, M: 4), ExtOp: Ops[0], IndexOp: Ops[5],
3245 ResTy: Ty, IntID: Intrinsic::aarch64_neon_tbx2, Name: "vtbx2");
3246 }
3247 case NEON::BI__builtin_neon_vqtbl1_v:
3248 case NEON::BI__builtin_neon_vqtbl1q_v:
3249 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
3250 case NEON::BI__builtin_neon_vqtbl2_v:
3251 case NEON::BI__builtin_neon_vqtbl2q_v: {
3252 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
3253 case NEON::BI__builtin_neon_vqtbl3_v:
3254 case NEON::BI__builtin_neon_vqtbl3q_v:
3255 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
3256 case NEON::BI__builtin_neon_vqtbl4_v:
3257 case NEON::BI__builtin_neon_vqtbl4q_v:
3258 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
3259 case NEON::BI__builtin_neon_vqtbx1_v:
3260 case NEON::BI__builtin_neon_vqtbx1q_v:
3261 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
3262 case NEON::BI__builtin_neon_vqtbx2_v:
3263 case NEON::BI__builtin_neon_vqtbx2q_v:
3264 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
3265 case NEON::BI__builtin_neon_vqtbx3_v:
3266 case NEON::BI__builtin_neon_vqtbx3q_v:
3267 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
3268 case NEON::BI__builtin_neon_vqtbx4_v:
3269 case NEON::BI__builtin_neon_vqtbx4q_v:
3270 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
3271 }
3272 }
3273
3274 if (!Int)
3275 return nullptr;
3276
3277 Function *F = CGF.CGM.getIntrinsic(IID: Int, Tys: Ty);
3278 return CGF.EmitNeonCall(F, Ops, name: s);
3279}
3280
3281Value *CodeGenFunction::vectorWrapScalar16(Value *Op) {
3282 auto *VTy = llvm::FixedVectorType::get(ElementType: Int16Ty, NumElts: 4);
3283 Op = Builder.CreateBitCast(V: Op, DestTy: Int16Ty);
3284 Value *V = PoisonValue::get(T: VTy);
3285 llvm::Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
3286 Op = Builder.CreateInsertElement(Vec: V, NewElt: Op, Idx: CI);
3287 return Op;
3288}
3289
3290/// SVEBuiltinMemEltTy - Returns the memory element type for this memory
3291/// access builtin. Only required if it can't be inferred from the base pointer
3292/// operand.
3293llvm::Type *CodeGenFunction::SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags) {
3294 switch (TypeFlags.getMemEltType()) {
3295 case SVETypeFlags::MemEltTyDefault:
3296 return getEltType(TypeFlags);
3297 case SVETypeFlags::MemEltTyInt8:
3298 return Builder.getInt8Ty();
3299 case SVETypeFlags::MemEltTyInt16:
3300 return Builder.getInt16Ty();
3301 case SVETypeFlags::MemEltTyInt32:
3302 return Builder.getInt32Ty();
3303 case SVETypeFlags::MemEltTyInt64:
3304 return Builder.getInt64Ty();
3305 }
3306 llvm_unreachable("Unknown MemEltType");
3307}
3308
3309llvm::Type *CodeGenFunction::getEltType(const SVETypeFlags &TypeFlags) {
3310 switch (TypeFlags.getEltType()) {
3311 default:
3312 llvm_unreachable("Invalid SVETypeFlag!");
3313
3314 case SVETypeFlags::EltTyMFloat8:
3315 case SVETypeFlags::EltTyInt8:
3316 return Builder.getInt8Ty();
3317 case SVETypeFlags::EltTyInt16:
3318 return Builder.getInt16Ty();
3319 case SVETypeFlags::EltTyInt32:
3320 return Builder.getInt32Ty();
3321 case SVETypeFlags::EltTyInt64:
3322 return Builder.getInt64Ty();
3323 case SVETypeFlags::EltTyInt128:
3324 return Builder.getInt128Ty();
3325
3326 case SVETypeFlags::EltTyFloat16:
3327 return Builder.getHalfTy();
3328 case SVETypeFlags::EltTyFloat32:
3329 return Builder.getFloatTy();
3330 case SVETypeFlags::EltTyFloat64:
3331 return Builder.getDoubleTy();
3332
3333 case SVETypeFlags::EltTyBFloat16:
3334 return Builder.getBFloatTy();
3335
3336 case SVETypeFlags::EltTyBool8:
3337 case SVETypeFlags::EltTyBool16:
3338 case SVETypeFlags::EltTyBool32:
3339 case SVETypeFlags::EltTyBool64:
3340 return Builder.getInt1Ty();
3341 }
3342}
3343
3344// Return the llvm predicate vector type corresponding to the specified element
3345// TypeFlags.
3346llvm::ScalableVectorType *
3347CodeGenFunction::getSVEPredType(const SVETypeFlags &TypeFlags) {
3348 switch (TypeFlags.getEltType()) {
3349 default: llvm_unreachable("Unhandled SVETypeFlag!");
3350
3351 case SVETypeFlags::EltTyInt8:
3352 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 16);
3353 case SVETypeFlags::EltTyInt16:
3354 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3355 case SVETypeFlags::EltTyInt32:
3356 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3357 case SVETypeFlags::EltTyInt64:
3358 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3359
3360 case SVETypeFlags::EltTyBFloat16:
3361 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3362 case SVETypeFlags::EltTyFloat16:
3363 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3364 case SVETypeFlags::EltTyFloat32:
3365 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3366 case SVETypeFlags::EltTyFloat64:
3367 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3368
3369 case SVETypeFlags::EltTyBool8:
3370 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 16);
3371 case SVETypeFlags::EltTyBool16:
3372 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3373 case SVETypeFlags::EltTyBool32:
3374 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3375 case SVETypeFlags::EltTyBool64:
3376 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3377 }
3378}
3379
3380// Return the llvm vector type corresponding to the specified element TypeFlags.
3381llvm::ScalableVectorType *
3382CodeGenFunction::getSVEType(const SVETypeFlags &TypeFlags) {
3383 switch (TypeFlags.getEltType()) {
3384 default:
3385 llvm_unreachable("Invalid SVETypeFlag!");
3386
3387 case SVETypeFlags::EltTyInt8:
3388 return llvm::ScalableVectorType::get(ElementType: Builder.getInt8Ty(), MinNumElts: 16);
3389 case SVETypeFlags::EltTyInt16:
3390 return llvm::ScalableVectorType::get(ElementType: Builder.getInt16Ty(), MinNumElts: 8);
3391 case SVETypeFlags::EltTyInt32:
3392 return llvm::ScalableVectorType::get(ElementType: Builder.getInt32Ty(), MinNumElts: 4);
3393 case SVETypeFlags::EltTyInt64:
3394 return llvm::ScalableVectorType::get(ElementType: Builder.getInt64Ty(), MinNumElts: 2);
3395
3396 case SVETypeFlags::EltTyMFloat8:
3397 return llvm::ScalableVectorType::get(ElementType: Builder.getInt8Ty(), MinNumElts: 16);
3398 case SVETypeFlags::EltTyFloat16:
3399 return llvm::ScalableVectorType::get(ElementType: Builder.getHalfTy(), MinNumElts: 8);
3400 case SVETypeFlags::EltTyBFloat16:
3401 return llvm::ScalableVectorType::get(ElementType: Builder.getBFloatTy(), MinNumElts: 8);
3402 case SVETypeFlags::EltTyFloat32:
3403 return llvm::ScalableVectorType::get(ElementType: Builder.getFloatTy(), MinNumElts: 4);
3404 case SVETypeFlags::EltTyFloat64:
3405 return llvm::ScalableVectorType::get(ElementType: Builder.getDoubleTy(), MinNumElts: 2);
3406
3407 case SVETypeFlags::EltTyBool8:
3408 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 16);
3409 case SVETypeFlags::EltTyBool16:
3410 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
3411 case SVETypeFlags::EltTyBool32:
3412 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
3413 case SVETypeFlags::EltTyBool64:
3414 return llvm::ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 2);
3415 }
3416}
3417
3418constexpr unsigned SVEBitsPerBlock = 128;
3419
3420static llvm::ScalableVectorType *getSVEVectorForElementType(llvm::Type *EltTy) {
3421 unsigned NumElts = SVEBitsPerBlock / EltTy->getScalarSizeInBits();
3422 return llvm::ScalableVectorType::get(ElementType: EltTy, MinNumElts: NumElts);
3423}
3424
3425// Reinterpret the input predicate so that it can be used to correctly isolate
3426// the elements of the specified datatype.
3427Value *CodeGenFunction::EmitSVEPredicateCast(Value *Pred,
3428 llvm::ScalableVectorType *VTy) {
3429
3430 if (isa<TargetExtType>(Val: Pred->getType()) &&
3431 cast<TargetExtType>(Val: Pred->getType())->getName() == "aarch64.svcount")
3432 return Pred;
3433
3434 auto *RTy = llvm::VectorType::get(ElementType: IntegerType::get(C&: getLLVMContext(), NumBits: 1), Other: VTy);
3435 if (Pred->getType() == RTy)
3436 return Pred;
3437
3438 unsigned IntID;
3439 llvm::Type *IntrinsicTy;
3440 switch (VTy->getMinNumElements()) {
3441 default:
3442 llvm_unreachable("unsupported element count!");
3443 case 1:
3444 case 2:
3445 case 4:
3446 case 8:
3447 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
3448 IntrinsicTy = RTy;
3449 break;
3450 case 16:
3451 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
3452 IntrinsicTy = Pred->getType();
3453 break;
3454 }
3455
3456 Function *F = CGM.getIntrinsic(IID: IntID, Tys: IntrinsicTy);
3457 Value *C = Builder.CreateCall(Callee: F, Args: Pred);
3458 assert(C->getType() == RTy && "Unexpected return type!");
3459 return C;
3460}
3461
3462Value *CodeGenFunction::EmitSVEPredicateTupleCast(Value *PredTuple,
3463 llvm::StructType *Ty) {
3464 if (PredTuple->getType() == Ty)
3465 return PredTuple;
3466
3467 Value *Ret = llvm::PoisonValue::get(T: Ty);
3468 for (unsigned I = 0; I < Ty->getNumElements(); ++I) {
3469 Value *Pred = Builder.CreateExtractValue(Agg: PredTuple, Idxs: I);
3470 Pred = EmitSVEPredicateCast(
3471 Pred, VTy: cast<llvm::ScalableVectorType>(Val: Ty->getTypeAtIndex(N: I)));
3472 Ret = Builder.CreateInsertValue(Agg: Ret, Val: Pred, Idxs: I);
3473 }
3474
3475 return Ret;
3476}
3477
3478Value *CodeGenFunction::EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
3479 SmallVectorImpl<Value *> &Ops,
3480 unsigned IntID) {
3481 auto *ResultTy = getSVEType(TypeFlags);
3482 auto *OverloadedTy =
3483 llvm::ScalableVectorType::get(ElementType: SVEBuiltinMemEltTy(TypeFlags), SVTy: ResultTy);
3484 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {OverloadedTy, Ops[1]->getType()});
3485
3486 // At the ACLE level there's only one predicate type, svbool_t, which is
3487 // mapped to <n x 16 x i1>. However, this might be incompatible with the
3488 // actual type being loaded. For example, when loading doubles (i64) the
3489 // predicate should be <n x 2 x i1> instead. At the IR level the type of
3490 // the predicate and the data being loaded must match. Cast to the type
3491 // expected by the intrinsic. The intrinsic itself should be defined in
3492 // a way than enforces relations between parameter types.
3493 Ops[0] = EmitSVEPredicateCast(
3494 Pred: Ops[0], VTy: cast<llvm::ScalableVectorType>(Val: F->getArg(i: 0)->getType()));
3495
3496 // Pass 0 when the offset is missing. This can only be applied when using
3497 // the "vector base" addressing mode for which ACLE allows no offset. The
3498 // corresponding LLVM IR always requires an offset.
3499 if (Ops.size() == 2) {
3500 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
3501 Ops.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: 0));
3502 }
3503
3504 // For "vector base, scalar index" scale the index so that it becomes a
3505 // scalar offset.
3506 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
3507 unsigned BytesPerElt =
3508 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
3509 Ops[2] = Builder.CreateShl(LHS: Ops[2], RHS: Log2_32(Value: BytesPerElt));
3510 }
3511
3512 Value *Call = Builder.CreateCall(Callee: F, Args: Ops);
3513
3514 // The following sext/zext is only needed when ResultTy != OverloadedTy. In
3515 // other cases it's folded into a nop.
3516 return TypeFlags.isZExtReturn() ? Builder.CreateZExt(V: Call, DestTy: ResultTy)
3517 : Builder.CreateSExt(V: Call, DestTy: ResultTy);
3518}
3519
3520Value *CodeGenFunction::EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
3521 SmallVectorImpl<Value *> &Ops,
3522 unsigned IntID) {
3523 auto *SrcDataTy = getSVEType(TypeFlags);
3524 auto *OverloadedTy =
3525 llvm::ScalableVectorType::get(ElementType: SVEBuiltinMemEltTy(TypeFlags), SVTy: SrcDataTy);
3526
3527 // In ACLE the source data is passed in the last argument, whereas in LLVM IR
3528 // it's the first argument. Move it accordingly.
3529 Ops.insert(I: Ops.begin(), Elt: Ops.pop_back_val());
3530
3531 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {OverloadedTy, Ops[2]->getType()});
3532
3533 // Pass 0 when the offset is missing. This can only be applied when using
3534 // the "vector base" addressing mode for which ACLE allows no offset. The
3535 // corresponding LLVM IR always requires an offset.
3536 if (Ops.size() == 3) {
3537 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
3538 Ops.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: 0));
3539 }
3540
3541 // Truncation is needed when SrcDataTy != OverloadedTy. In other cases it's
3542 // folded into a nop.
3543 Ops[0] = Builder.CreateTrunc(V: Ops[0], DestTy: OverloadedTy);
3544
3545 // At the ACLE level there's only one predicate type, svbool_t, which is
3546 // mapped to <n x 16 x i1>. However, this might be incompatible with the
3547 // actual type being stored. For example, when storing doubles (i64) the
3548 // predicated should be <n x 2 x i1> instead. At the IR level the type of
3549 // the predicate and the data being stored must match. Cast to the type
3550 // expected by the intrinsic. The intrinsic itself should be defined in
3551 // a way that enforces relations between parameter types.
3552 Ops[1] = EmitSVEPredicateCast(
3553 Pred: Ops[1], VTy: cast<llvm::ScalableVectorType>(Val: F->getArg(i: 1)->getType()));
3554
3555 // For "vector base, scalar index" scale the index so that it becomes a
3556 // scalar offset.
3557 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
3558 unsigned BytesPerElt =
3559 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
3560 Ops[3] = Builder.CreateShl(LHS: Ops[3], RHS: Log2_32(Value: BytesPerElt));
3561 }
3562
3563 return Builder.CreateCall(Callee: F, Args: Ops);
3564}
3565
3566Value *CodeGenFunction::EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
3567 SmallVectorImpl<Value *> &Ops,
3568 unsigned IntID) {
3569 // The gather prefetches are overloaded on the vector input - this can either
3570 // be the vector of base addresses or vector of offsets.
3571 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Val: Ops[1]->getType());
3572 if (!OverloadedTy)
3573 OverloadedTy = cast<llvm::ScalableVectorType>(Val: Ops[2]->getType());
3574
3575 // Cast the predicate from svbool_t to the right number of elements.
3576 Ops[0] = EmitSVEPredicateCast(Pred: Ops[0], VTy: OverloadedTy);
3577
3578 // vector + imm addressing modes
3579 if (Ops[1]->getType()->isVectorTy()) {
3580 if (Ops.size() == 3) {
3581 // Pass 0 for 'vector+imm' when the index is omitted.
3582 Ops.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: 0));
3583
3584 // The sv_prfop is the last operand in the builtin and IR intrinsic.
3585 std::swap(a&: Ops[2], b&: Ops[3]);
3586 } else {
3587 // Index needs to be passed as scaled offset.
3588 llvm::Type *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
3589 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
3590 if (BytesPerElt > 1)
3591 Ops[2] = Builder.CreateShl(LHS: Ops[2], RHS: Log2_32(Value: BytesPerElt));
3592 }
3593
3594 Function *F = CGM.getIntrinsic(IID: IntID, Tys: OverloadedTy);
3595 return Builder.CreateCall(Callee: F, Args: Ops);
3596 }
3597
3598 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {Ops[1]->getType(), OverloadedTy});
3599 return Builder.CreateCall(Callee: F, Args: Ops);
3600}
3601
3602Value *CodeGenFunction::EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
3603 SmallVectorImpl<Value*> &Ops,
3604 unsigned IntID) {
3605 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
3606 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy);
3607 Value *BasePtr = Ops[1];
3608
3609 // Does the load have an offset?
3610 if (Ops.size() > 2)
3611 BasePtr = Builder.CreateGEP(Ty: VTy, Ptr: BasePtr, IdxList: Ops[2]);
3612
3613 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {VTy, BasePtr->getType()});
3614 return Builder.CreateCall(Callee: F, Args: {Predicate, BasePtr});
3615}
3616
3617Value *CodeGenFunction::EmitSVEStructStore(const SVETypeFlags &TypeFlags,
3618 SmallVectorImpl<Value*> &Ops,
3619 unsigned IntID) {
3620 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
3621
3622 unsigned N;
3623 switch (IntID) {
3624 case Intrinsic::aarch64_sve_st2:
3625 case Intrinsic::aarch64_sve_st1_pn_x2:
3626 case Intrinsic::aarch64_sve_stnt1_pn_x2:
3627 case Intrinsic::aarch64_sve_st2q:
3628 N = 2;
3629 break;
3630 case Intrinsic::aarch64_sve_st3:
3631 case Intrinsic::aarch64_sve_st3q:
3632 N = 3;
3633 break;
3634 case Intrinsic::aarch64_sve_st4:
3635 case Intrinsic::aarch64_sve_st1_pn_x4:
3636 case Intrinsic::aarch64_sve_stnt1_pn_x4:
3637 case Intrinsic::aarch64_sve_st4q:
3638 N = 4;
3639 break;
3640 default:
3641 llvm_unreachable("unknown intrinsic!");
3642 }
3643
3644 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy);
3645 Value *BasePtr = Ops[1];
3646
3647 // Does the store have an offset?
3648 if (Ops.size() > (2 + N))
3649 BasePtr = Builder.CreateGEP(Ty: VTy, Ptr: BasePtr, IdxList: Ops[2]);
3650
3651 // The llvm.aarch64.sve.st2/3/4 intrinsics take legal part vectors, so we
3652 // need to break up the tuple vector.
3653 SmallVector<llvm::Value*, 5> Operands;
3654 for (unsigned I = Ops.size() - N; I < Ops.size(); ++I)
3655 Operands.push_back(Elt: Ops[I]);
3656 Operands.append(IL: {Predicate, BasePtr});
3657 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {VTy, BasePtr->getType()});
3658
3659 return Builder.CreateCall(Callee: F, Args: Operands);
3660}
3661
3662// SVE2's svpmullb and svpmullt builtins are similar to the svpmullb_pair and
3663// svpmullt_pair intrinsics, with the exception that their results are bitcast
3664// to a wider type.
3665Value *CodeGenFunction::EmitSVEPMull(const SVETypeFlags &TypeFlags,
3666 SmallVectorImpl<Value *> &Ops,
3667 unsigned BuiltinID) {
3668 // Splat scalar operand to vector (intrinsics with _n infix)
3669 if (TypeFlags.hasSplatOperand()) {
3670 unsigned OpNo = TypeFlags.getSplatOperand();
3671 Ops[OpNo] = EmitSVEDupX(Scalar: Ops[OpNo]);
3672 }
3673
3674 // The pair-wise function has a narrower overloaded type.
3675 Function *F = CGM.getIntrinsic(IID: BuiltinID, Tys: Ops[0]->getType());
3676 Value *Call = Builder.CreateCall(Callee: F, Args: {Ops[0], Ops[1]});
3677
3678 // Now bitcast to the wider result type.
3679 llvm::ScalableVectorType *Ty = getSVEType(TypeFlags);
3680 return EmitSVEReinterpret(Val: Call, Ty);
3681}
3682
3683Value *CodeGenFunction::EmitSVEMovl(const SVETypeFlags &TypeFlags,
3684 ArrayRef<Value *> Ops, unsigned BuiltinID) {
3685 llvm::Type *OverloadedTy = getSVEType(TypeFlags);
3686 Function *F = CGM.getIntrinsic(IID: BuiltinID, Tys: OverloadedTy);
3687 return Builder.CreateCall(Callee: F, Args: {Ops[0], Builder.getInt32(C: 0)});
3688}
3689
3690Value *CodeGenFunction::EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
3691 SmallVectorImpl<Value *> &Ops,
3692 unsigned BuiltinID) {
3693 auto *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
3694 auto *VectorTy = getSVEVectorForElementType(EltTy: MemEltTy);
3695 auto *MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, SVTy: VectorTy);
3696
3697 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy: MemoryTy);
3698 Value *BasePtr = Ops[1];
3699
3700 // Implement the index operand if not omitted.
3701 if (Ops.size() > 3)
3702 BasePtr = Builder.CreateGEP(Ty: MemoryTy, Ptr: BasePtr, IdxList: Ops[2]);
3703
3704 Value *PrfOp = Ops.back();
3705
3706 llvm::Type *Tys[2] = {Predicate->getType(), BasePtr->getType()};
3707 Function *F = CGM.getIntrinsic(IID: BuiltinID, Tys);
3708 return Builder.CreateCall(Callee: F, Args: {Predicate, BasePtr, PrfOp});
3709}
3710
3711Value *CodeGenFunction::EmitSVEMaskedLoad(const CallExpr *E,
3712 llvm::Type *ReturnTy,
3713 SmallVectorImpl<Value *> &Ops,
3714 unsigned IntrinsicID,
3715 bool IsZExtReturn) {
3716 QualType LangPTy = E->getArg(Arg: 1)->getType();
3717 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
3718 T: LangPTy->castAs<PointerType>()->getPointeeType());
3719
3720 // Mfloat8 types is stored as a vector, so extra work
3721 // to extract sclar element type is necessary.
3722 if (MemEltTy->isVectorTy()) {
3723 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
3724 "Only <1 x i8> expected");
3725 MemEltTy = cast<llvm::VectorType>(Val: MemEltTy)->getElementType();
3726 }
3727
3728 // The vector type that is returned may be different from the
3729 // eventual type loaded from memory.
3730 auto VectorTy = cast<llvm::ScalableVectorType>(Val: ReturnTy);
3731 llvm::ScalableVectorType *MemoryTy = nullptr;
3732 llvm::ScalableVectorType *PredTy = nullptr;
3733 bool IsQuadLoad = false;
3734 switch (IntrinsicID) {
3735 case Intrinsic::aarch64_sve_ld1uwq:
3736 case Intrinsic::aarch64_sve_ld1udq:
3737 MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, MinNumElts: 1);
3738 PredTy = llvm::ScalableVectorType::get(
3739 ElementType: llvm::Type::getInt1Ty(C&: getLLVMContext()), MinNumElts: 1);
3740 IsQuadLoad = true;
3741 break;
3742 default:
3743 MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, SVTy: VectorTy);
3744 PredTy = MemoryTy;
3745 break;
3746 }
3747
3748 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy: PredTy);
3749 Value *BasePtr = Ops[1];
3750
3751 // Does the load have an offset?
3752 if (Ops.size() > 2)
3753 BasePtr = Builder.CreateGEP(Ty: MemoryTy, Ptr: BasePtr, IdxList: Ops[2]);
3754
3755 llvm::Type *Tys[2] = {IsQuadLoad ? VectorTy : MemoryTy, BasePtr->getType()};
3756 Function *F = CGM.getIntrinsic(IID: IntrinsicID, Tys);
3757 auto *Load = Builder.CreateCall(Callee: F, Args: {Predicate, BasePtr});
3758 auto TBAAInfo = CGM.getTBAAAccessInfo(AccessType: LangPTy->getPointeeType());
3759 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo);
3760
3761 if (IsQuadLoad)
3762 return Load;
3763
3764 return IsZExtReturn ? Builder.CreateZExt(V: Load, DestTy: VectorTy)
3765 : Builder.CreateSExt(V: Load, DestTy: VectorTy);
3766}
3767
3768Value *CodeGenFunction::EmitSVEMaskedStore(const CallExpr *E,
3769 SmallVectorImpl<Value *> &Ops,
3770 unsigned IntrinsicID) {
3771 QualType LangPTy = E->getArg(Arg: 1)->getType();
3772 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
3773 T: LangPTy->castAs<PointerType>()->getPointeeType());
3774
3775 // Mfloat8 types is stored as a vector, so extra work
3776 // to extract sclar element type is necessary.
3777 if (MemEltTy->isVectorTy()) {
3778 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
3779 "Only <1 x i8> expected");
3780 MemEltTy = cast<llvm::VectorType>(Val: MemEltTy)->getElementType();
3781 }
3782
3783 // The vector type that is stored may be different from the
3784 // eventual type stored to memory.
3785 auto VectorTy = cast<llvm::ScalableVectorType>(Val: Ops.back()->getType());
3786 auto MemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, SVTy: VectorTy);
3787
3788 auto PredTy = MemoryTy;
3789 auto AddrMemoryTy = MemoryTy;
3790 bool IsQuadStore = false;
3791
3792 switch (IntrinsicID) {
3793 case Intrinsic::aarch64_sve_st1wq:
3794 case Intrinsic::aarch64_sve_st1dq:
3795 AddrMemoryTy = llvm::ScalableVectorType::get(ElementType: MemEltTy, MinNumElts: 1);
3796 PredTy =
3797 llvm::ScalableVectorType::get(ElementType: IntegerType::get(C&: getLLVMContext(), NumBits: 1), MinNumElts: 1);
3798 IsQuadStore = true;
3799 break;
3800 default:
3801 break;
3802 }
3803 Value *Predicate = EmitSVEPredicateCast(Pred: Ops[0], VTy: PredTy);
3804 Value *BasePtr = Ops[1];
3805
3806 // Does the store have an offset?
3807 if (Ops.size() == 4)
3808 BasePtr = Builder.CreateGEP(Ty: AddrMemoryTy, Ptr: BasePtr, IdxList: Ops[2]);
3809
3810 // Last value is always the data
3811 Value *Val =
3812 IsQuadStore ? Ops.back() : Builder.CreateTrunc(V: Ops.back(), DestTy: MemoryTy);
3813
3814 llvm::Type *Tys[2] = {IsQuadStore ? VectorTy : MemoryTy, BasePtr->getType()};
3815 Function *F = CGM.getIntrinsic(IID: IntrinsicID, Tys);
3816 auto *Store = Builder.CreateCall(Callee: F, Args: {Val, Predicate, BasePtr});
3817 auto TBAAInfo = CGM.getTBAAAccessInfo(AccessType: LangPTy->getPointeeType());
3818 CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
3819 return Store;
3820}
3821
3822Value *CodeGenFunction::EmitSMELd1St1(const SVETypeFlags &TypeFlags,
3823 SmallVectorImpl<Value *> &Ops,
3824 unsigned IntID) {
3825 Ops[2] = EmitSVEPredicateCast(
3826 Pred: Ops[2], VTy: getSVEVectorForElementType(EltTy: SVEBuiltinMemEltTy(TypeFlags)));
3827
3828 SmallVector<Value *> NewOps;
3829 NewOps.push_back(Elt: Ops[2]);
3830
3831 llvm::Value *BasePtr = Ops[3];
3832 llvm::Value *RealSlice = Ops[1];
3833 // If the intrinsic contains the vnum parameter, multiply it with the vector
3834 // size in bytes.
3835 if (Ops.size() == 5) {
3836 Function *StreamingVectorLength =
3837 CGM.getIntrinsic(IID: Intrinsic::aarch64_sme_cntsd);
3838 llvm::Value *StreamingVectorLengthCall =
3839 Builder.CreateMul(LHS: Builder.CreateCall(Callee: StreamingVectorLength),
3840 RHS: llvm::ConstantInt::get(Ty: Int64Ty, V: 8), Name: "svl",
3841 /* HasNUW */ true, /* HasNSW */ true);
3842 llvm::Value *Mulvl =
3843 Builder.CreateMul(LHS: StreamingVectorLengthCall, RHS: Ops[4], Name: "mulvl");
3844 // The type of the ptr parameter is void *, so use Int8Ty here.
3845 BasePtr = Builder.CreateGEP(Ty: Int8Ty, Ptr: Ops[3], IdxList: Mulvl);
3846 RealSlice = Builder.CreateZExt(V: RealSlice, DestTy: Int64Ty);
3847 RealSlice = Builder.CreateAdd(LHS: RealSlice, RHS: Ops[4]);
3848 RealSlice = Builder.CreateTrunc(V: RealSlice, DestTy: Int32Ty);
3849 }
3850 NewOps.push_back(Elt: BasePtr);
3851 NewOps.push_back(Elt: Ops[0]);
3852 NewOps.push_back(Elt: RealSlice);
3853 Function *F = CGM.getIntrinsic(IID: IntID, Tys: BasePtr->getType());
3854 return Builder.CreateCall(Callee: F, Args: NewOps);
3855}
3856
3857Value *CodeGenFunction::EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
3858 SmallVectorImpl<Value *> &Ops,
3859 unsigned IntID) {
3860 auto *VecTy = getSVEType(TypeFlags);
3861 Function *F = CGM.getIntrinsic(IID: IntID, Tys: VecTy);
3862 if (TypeFlags.isReadZA())
3863 Ops[1] = EmitSVEPredicateCast(Pred: Ops[1], VTy: VecTy);
3864 else if (TypeFlags.isWriteZA())
3865 Ops[2] = EmitSVEPredicateCast(Pred: Ops[2], VTy: VecTy);
3866 return Builder.CreateCall(Callee: F, Args: Ops);
3867}
3868
3869Value *CodeGenFunction::EmitSMEZero(const SVETypeFlags &TypeFlags,
3870 SmallVectorImpl<Value *> &Ops,
3871 unsigned IntID) {
3872 // svzero_za() intrinsic zeros the entire za tile and has no paramters.
3873 if (Ops.size() == 0)
3874 Ops.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: 255));
3875 Function *F = CGM.getIntrinsic(IID: IntID, Tys: {});
3876 return Builder.CreateCall(Callee: F, Args: Ops);
3877}
3878
3879Value *CodeGenFunction::EmitSMELdrStr(const SVETypeFlags &TypeFlags,
3880 SmallVectorImpl<Value *> &Ops,
3881 unsigned IntID) {
3882 if (Ops.size() == 2)
3883 Ops.push_back(Elt: Builder.getInt32(C: 0));
3884 else
3885 Ops[2] = Builder.CreateIntCast(V: Ops[2], DestTy: Int32Ty, isSigned: true);
3886 Function *F = CGM.getIntrinsic(IID: IntID, Tys: Ops[1]->getType());
3887 return Builder.CreateCall(Callee: F, Args: Ops);
3888}
3889
3890// Limit the usage of scalable llvm IR generated by the ACLE by using the
3891// sve dup.x intrinsic instead of IRBuilder::CreateVectorSplat.
3892Value *CodeGenFunction::EmitSVEDupX(Value *Scalar, llvm::Type *Ty) {
3893 return Builder.CreateVectorSplat(
3894 EC: cast<llvm::VectorType>(Val: Ty)->getElementCount(), V: Scalar);
3895}
3896
3897Value *CodeGenFunction::EmitSVEDupX(Value *Scalar) {
3898 if (auto *Ty = Scalar->getType(); Ty->isVectorTy()) {
3899#ifndef NDEBUG
3900 auto *VecTy = cast<llvm::VectorType>(Ty);
3901 ElementCount EC = VecTy->getElementCount();
3902 assert(EC.isScalar() && VecTy->getElementType() == Int8Ty &&
3903 "Only <1 x i8> expected");
3904#endif
3905 Scalar = Builder.CreateExtractElement(Vec: Scalar, Idx: uint64_t(0));
3906 }
3907 return EmitSVEDupX(Scalar, Ty: getSVEVectorForElementType(EltTy: Scalar->getType()));
3908}
3909
3910Value *CodeGenFunction::EmitSVEReinterpret(Value *Val, llvm::Type *Ty) {
3911 // FIXME: For big endian this needs an additional REV, or needs a separate
3912 // intrinsic that is code-generated as a no-op, because the LLVM bitcast
3913 // instruction is defined as 'bitwise' equivalent from memory point of
3914 // view (when storing/reloading), whereas the svreinterpret builtin
3915 // implements bitwise equivalent cast from register point of view.
3916 // LLVM CodeGen for a bitcast must add an explicit REV for big-endian.
3917
3918 if (auto *StructTy = dyn_cast<StructType>(Val: Ty)) {
3919 Value *Tuple = llvm::PoisonValue::get(T: Ty);
3920
3921 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
3922 Value *In = Builder.CreateExtractValue(Agg: Val, Idxs: I);
3923 Value *Out = Builder.CreateBitCast(V: In, DestTy: StructTy->getTypeAtIndex(N: I));
3924 Tuple = Builder.CreateInsertValue(Agg: Tuple, Val: Out, Idxs: I);
3925 }
3926
3927 return Tuple;
3928 }
3929
3930 return Builder.CreateBitCast(V: Val, DestTy: Ty);
3931}
3932
3933static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty,
3934 SmallVectorImpl<Value *> &Ops) {
3935 auto *SplatZero = Constant::getNullValue(Ty);
3936 Ops.insert(I: Ops.begin(), Elt: SplatZero);
3937}
3938
3939static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty,
3940 SmallVectorImpl<Value *> &Ops) {
3941 auto *SplatUndef = UndefValue::get(T: Ty);
3942 Ops.insert(I: Ops.begin(), Elt: SplatUndef);
3943}
3944
3945SmallVector<llvm::Type *, 2>
3946CodeGenFunction::getSVEOverloadTypes(const SVETypeFlags &TypeFlags,
3947 llvm::Type *ResultType,
3948 ArrayRef<Value *> Ops) {
3949 if (TypeFlags.isOverloadNone())
3950 return {};
3951
3952 llvm::Type *DefaultType = getSVEType(TypeFlags);
3953
3954 if (TypeFlags.isOverloadWhileOrMultiVecCvt())
3955 return {DefaultType, Ops[1]->getType()};
3956
3957 if (TypeFlags.isOverloadWhileRW())
3958 return {getSVEPredType(TypeFlags), Ops[0]->getType()};
3959
3960 if (TypeFlags.isOverloadDefaultAndOp0())
3961 return {DefaultType, Ops[0]->getType()};
3962
3963 if (TypeFlags.isOverloadFirstandLast())
3964 return {Ops[0]->getType(), Ops.back()->getType()};
3965
3966 if (TypeFlags.isReductionQV())
3967 return {ResultType, Ops[1]->getType()};
3968
3969 assert(TypeFlags.isOverloadDefault() && "Unexpected value for overloads");
3970 return {DefaultType};
3971}
3972
3973Value *CodeGenFunction::EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
3974 ArrayRef<Value *> Ops) {
3975 assert((TypeFlags.isTupleSet() || TypeFlags.isTupleGet()) &&
3976 "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()");
3977 unsigned Idx = cast<ConstantInt>(Val: Ops[1])->getZExtValue();
3978
3979 if (TypeFlags.isTupleSet())
3980 return Builder.CreateInsertValue(Agg: Ops[0], Val: Ops[2], Idxs: Idx);
3981 return Builder.CreateExtractValue(Agg: Ops[0], Idxs: Idx);
3982}
3983
3984Value *CodeGenFunction::EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
3985 llvm::Type *Ty,
3986 ArrayRef<Value *> Ops) {
3987 assert(TypeFlags.isTupleCreate() && "Expects TypleFlag isTupleCreate");
3988
3989 Value *Tuple = llvm::PoisonValue::get(T: Ty);
3990 for (unsigned Idx = 0; Idx < Ops.size(); Idx++)
3991 Tuple = Builder.CreateInsertValue(Agg: Tuple, Val: Ops[Idx], Idxs: Idx);
3992
3993 return Tuple;
3994}
3995
3996void CodeGenFunction::GetAArch64SVEProcessedOperands(
3997 unsigned BuiltinID, const CallExpr *E, SmallVectorImpl<Value *> &Ops,
3998 SVETypeFlags TypeFlags) {
3999 // Find out if any arguments are required to be integer constant expressions.
4000 unsigned ICEArguments = 0;
4001 ASTContext::GetBuiltinTypeError Error;
4002 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
4003 assert(Error == ASTContext::GE_None && "Should not codegen an error");
4004
4005 // Tuple set/get only requires one insert/extract vector, which is
4006 // created by EmitSVETupleSetOrGet.
4007 bool IsTupleGetOrSet = TypeFlags.isTupleSet() || TypeFlags.isTupleGet();
4008
4009 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
4010 bool IsICE = ICEArguments & (1 << i);
4011 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: i));
4012
4013 if (IsICE) {
4014 // If this is required to be a constant, constant fold it so that we know
4015 // that the generated intrinsic gets a ConstantInt.
4016 std::optional<llvm::APSInt> Result =
4017 E->getArg(Arg: i)->getIntegerConstantExpr(Ctx: getContext());
4018 assert(Result && "Expected argument to be a constant");
4019
4020 // Immediates for SVE llvm intrinsics are always 32bit. We can safely
4021 // truncate because the immediate has been range checked and no valid
4022 // immediate requires more than a handful of bits.
4023 *Result = Result->extOrTrunc(width: 32);
4024 Ops.push_back(Elt: llvm::ConstantInt::get(Context&: getLLVMContext(), V: *Result));
4025 continue;
4026 }
4027
4028 if (isa<StructType>(Val: Arg->getType()) && !IsTupleGetOrSet) {
4029 for (unsigned I = 0; I < Arg->getType()->getStructNumElements(); ++I)
4030 Ops.push_back(Elt: Builder.CreateExtractValue(Agg: Arg, Idxs: I));
4031
4032 continue;
4033 }
4034
4035 Ops.push_back(Elt: Arg);
4036 }
4037}
4038
4039Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
4040 const CallExpr *E) {
4041 llvm::Type *Ty = ConvertType(T: E->getType());
4042 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
4043 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
4044 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
4045 return EmitSVEReinterpret(Val, Ty);
4046 }
4047
4048 auto *Builtin =
4049 findARMVectorIntrinsicInMap(IntrinsicMap: ArrayRef(AArch64SVEIntrinsicMap), BuiltinID,
4050 MapProvenSorted&: AArch64SVEIntrinsicsProvenSorted);
4051
4052 llvm::SmallVector<Value *, 4> Ops;
4053 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4054 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
4055
4056 if (TypeFlags.isLoad())
4057 return EmitSVEMaskedLoad(E, ReturnTy: Ty, Ops, IntrinsicID: Builtin->LLVMIntrinsic,
4058 IsZExtReturn: TypeFlags.isZExtReturn());
4059 if (TypeFlags.isStore())
4060 return EmitSVEMaskedStore(E, Ops, IntrinsicID: Builtin->LLVMIntrinsic);
4061 if (TypeFlags.isGatherLoad())
4062 return EmitSVEGatherLoad(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4063 if (TypeFlags.isScatterStore())
4064 return EmitSVEScatterStore(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4065 if (TypeFlags.isPrefetch())
4066 return EmitSVEPrefetchLoad(TypeFlags, Ops, BuiltinID: Builtin->LLVMIntrinsic);
4067 if (TypeFlags.isGatherPrefetch())
4068 return EmitSVEGatherPrefetch(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4069 if (TypeFlags.isStructLoad())
4070 return EmitSVEStructLoad(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4071 if (TypeFlags.isStructStore())
4072 return EmitSVEStructStore(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4073 if (TypeFlags.isTupleSet() || TypeFlags.isTupleGet())
4074 return EmitSVETupleSetOrGet(TypeFlags, Ops);
4075 if (TypeFlags.isTupleCreate())
4076 return EmitSVETupleCreate(TypeFlags, Ty, Ops);
4077 if (TypeFlags.isUndef())
4078 return UndefValue::get(T: Ty);
4079
4080 // Handle built-ins for which there is a corresponding LLVM Intrinsic.
4081 // -------------------------------------------------------------------
4082 if (Builtin->LLVMIntrinsic != 0) {
4083 // Emit set FPMR for intrinsics that require it
4084 if (TypeFlags.setsFPMR())
4085 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_set_fpmr),
4086 Args: Ops.pop_back_val());
4087 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZeroExp)
4088 InsertExplicitZeroOperand(Builder, Ty, Ops);
4089
4090 if (TypeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
4091 InsertExplicitUndefOperand(Builder, Ty, Ops);
4092
4093 // Some ACLE builtins leave out the argument to specify the predicate
4094 // pattern, which is expected to be expanded to an SV_ALL pattern.
4095 if (TypeFlags.isAppendSVALL())
4096 Ops.push_back(Elt: Builder.getInt32(/*SV_ALL*/ C: 31));
4097 if (TypeFlags.isInsertOp1SVALL())
4098 Ops.insert(I: &Ops[1], Elt: Builder.getInt32(/*SV_ALL*/ C: 31));
4099
4100 // Predicates must match the main datatype.
4101 for (Value *&Op : Ops)
4102 if (auto PredTy = dyn_cast<llvm::VectorType>(Val: Op->getType()))
4103 if (PredTy->getElementType()->isIntegerTy(BitWidth: 1))
4104 Op = EmitSVEPredicateCast(Pred: Op, VTy: getSVEType(TypeFlags));
4105
4106 // Splat scalar operand to vector (intrinsics with _n infix)
4107 if (TypeFlags.hasSplatOperand()) {
4108 unsigned OpNo = TypeFlags.getSplatOperand();
4109 Ops[OpNo] = EmitSVEDupX(Scalar: Ops[OpNo]);
4110 }
4111
4112 if (TypeFlags.isReverseCompare())
4113 std::swap(a&: Ops[1], b&: Ops[2]);
4114 else if (TypeFlags.isReverseUSDOT())
4115 std::swap(a&: Ops[1], b&: Ops[2]);
4116 else if (TypeFlags.isReverseMergeAnyBinOp() &&
4117 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4118 std::swap(a&: Ops[1], b&: Ops[2]);
4119 else if (TypeFlags.isReverseMergeAnyAccOp() &&
4120 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4121 std::swap(a&: Ops[1], b&: Ops[3]);
4122
4123 // Predicated intrinsics with _z suffix need a select w/ zeroinitializer.
4124 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZero) {
4125 llvm::Type *OpndTy = Ops[1]->getType();
4126 auto *SplatZero = Constant::getNullValue(Ty: OpndTy);
4127 Ops[1] = Builder.CreateSelect(C: Ops[0], True: Ops[1], False: SplatZero);
4128 }
4129
4130 Function *F = CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic,
4131 Tys: getSVEOverloadTypes(TypeFlags, ResultType: Ty, Ops));
4132 Value *Call = Builder.CreateCall(Callee: F, Args: Ops);
4133
4134 if (Call->getType() == Ty)
4135 return Call;
4136
4137 // Predicate results must be converted to svbool_t.
4138 if (auto PredTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty))
4139 return EmitSVEPredicateCast(Pred: Call, VTy: PredTy);
4140 if (auto PredTupleTy = dyn_cast<llvm::StructType>(Val: Ty))
4141 return EmitSVEPredicateTupleCast(PredTuple: Call, Ty: PredTupleTy);
4142
4143 llvm_unreachable("unsupported element count!");
4144 }
4145
4146 switch (BuiltinID) {
4147 default:
4148 return nullptr;
4149
4150 case SVE::BI__builtin_sve_svreinterpret_b: {
4151 Function *CastFromSVCountF =
4152 CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_convert_from_svcount);
4153 return Builder.CreateCall(Callee: CastFromSVCountF, Args: Ops[0]);
4154 }
4155 case SVE::BI__builtin_sve_svreinterpret_c: {
4156 Function *CastToSVCountF =
4157 CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_convert_to_svcount);
4158 return Builder.CreateCall(Callee: CastToSVCountF, Args: Ops[0]);
4159 }
4160
4161 case SVE::BI__builtin_sve_svpsel_lane_b8:
4162 case SVE::BI__builtin_sve_svpsel_lane_b16:
4163 case SVE::BI__builtin_sve_svpsel_lane_b32:
4164 case SVE::BI__builtin_sve_svpsel_lane_b64:
4165 case SVE::BI__builtin_sve_svpsel_lane_c8:
4166 case SVE::BI__builtin_sve_svpsel_lane_c16:
4167 case SVE::BI__builtin_sve_svpsel_lane_c32:
4168 case SVE::BI__builtin_sve_svpsel_lane_c64: {
4169 auto OverloadedTy = getSVEType(TypeFlags: SVETypeFlags(Builtin->TypeModifier));
4170 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_psel,
4171 Tys: {Ops[0]->getType(), OverloadedTy});
4172 llvm::Value *Ops1 = EmitSVEPredicateCast(Pred: Ops[1], VTy: OverloadedTy);
4173 return Builder.CreateCall(Callee: F, Args: {Ops[0], Ops1, Ops[2]});
4174 }
4175 case SVE::BI__builtin_sve_svmov_b_z: {
4176 // svmov_b_z(pg, op) <=> svand_b_z(pg, op, op)
4177 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4178 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4179 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_and_z, Tys: OverloadedTy);
4180 return Builder.CreateCall(Callee: F, Args: {Ops[0], Ops[1], Ops[1]});
4181 }
4182
4183 case SVE::BI__builtin_sve_svnot_b_z: {
4184 // svnot_b_z(pg, op) <=> sveor_b_z(pg, op, pg)
4185 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4186 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4187 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_eor_z, Tys: OverloadedTy);
4188 return Builder.CreateCall(Callee: F, Args: {Ops[0], Ops[1], Ops[0]});
4189 }
4190
4191 case SVE::BI__builtin_sve_svmovlb_u16:
4192 case SVE::BI__builtin_sve_svmovlb_u32:
4193 case SVE::BI__builtin_sve_svmovlb_u64:
4194 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_ushllb);
4195
4196 case SVE::BI__builtin_sve_svmovlb_s16:
4197 case SVE::BI__builtin_sve_svmovlb_s32:
4198 case SVE::BI__builtin_sve_svmovlb_s64:
4199 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_sshllb);
4200
4201 case SVE::BI__builtin_sve_svmovlt_u16:
4202 case SVE::BI__builtin_sve_svmovlt_u32:
4203 case SVE::BI__builtin_sve_svmovlt_u64:
4204 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_ushllt);
4205
4206 case SVE::BI__builtin_sve_svmovlt_s16:
4207 case SVE::BI__builtin_sve_svmovlt_s32:
4208 case SVE::BI__builtin_sve_svmovlt_s64:
4209 return EmitSVEMovl(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_sshllt);
4210
4211 case SVE::BI__builtin_sve_svpmullt_u16:
4212 case SVE::BI__builtin_sve_svpmullt_u64:
4213 case SVE::BI__builtin_sve_svpmullt_n_u16:
4214 case SVE::BI__builtin_sve_svpmullt_n_u64:
4215 return EmitSVEPMull(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_pmullt_pair);
4216
4217 case SVE::BI__builtin_sve_svpmullb_u16:
4218 case SVE::BI__builtin_sve_svpmullb_u64:
4219 case SVE::BI__builtin_sve_svpmullb_n_u16:
4220 case SVE::BI__builtin_sve_svpmullb_n_u64:
4221 return EmitSVEPMull(TypeFlags, Ops, BuiltinID: Intrinsic::aarch64_sve_pmullb_pair);
4222
4223 case SVE::BI__builtin_sve_svdup_n_b8:
4224 case SVE::BI__builtin_sve_svdup_n_b16:
4225 case SVE::BI__builtin_sve_svdup_n_b32:
4226 case SVE::BI__builtin_sve_svdup_n_b64: {
4227 llvm::ScalableVectorType *OverloadedTy = getSVEType(TypeFlags);
4228 Value *Dup = EmitSVEDupX(Scalar: Ops[0], Ty: OverloadedTy);
4229 return EmitSVEPredicateCast(Pred: Dup, VTy: cast<llvm::ScalableVectorType>(Val: Ty));
4230 }
4231
4232 case SVE::BI__builtin_sve_svdupq_n_b8:
4233 case SVE::BI__builtin_sve_svdupq_n_b16:
4234 case SVE::BI__builtin_sve_svdupq_n_b32:
4235 case SVE::BI__builtin_sve_svdupq_n_b64:
4236 case SVE::BI__builtin_sve_svdupq_n_u8:
4237 case SVE::BI__builtin_sve_svdupq_n_s8:
4238 case SVE::BI__builtin_sve_svdupq_n_u64:
4239 case SVE::BI__builtin_sve_svdupq_n_f64:
4240 case SVE::BI__builtin_sve_svdupq_n_s64:
4241 case SVE::BI__builtin_sve_svdupq_n_u16:
4242 case SVE::BI__builtin_sve_svdupq_n_f16:
4243 case SVE::BI__builtin_sve_svdupq_n_bf16:
4244 case SVE::BI__builtin_sve_svdupq_n_s16:
4245 case SVE::BI__builtin_sve_svdupq_n_u32:
4246 case SVE::BI__builtin_sve_svdupq_n_f32:
4247 case SVE::BI__builtin_sve_svdupq_n_s32: {
4248 // These builtins are implemented by storing each element to an array and using
4249 // ld1rq to materialize a vector.
4250 unsigned NumOpnds = Ops.size();
4251
4252 bool IsBoolTy =
4253 cast<llvm::VectorType>(Val: Ty)->getElementType()->isIntegerTy(BitWidth: 1);
4254
4255 // For svdupq_n_b* the element type of is an integer of type 128/numelts,
4256 // so that the compare can use the width that is natural for the expected
4257 // number of predicate lanes.
4258 llvm::Type *EltTy = Ops[0]->getType();
4259 if (IsBoolTy)
4260 EltTy = IntegerType::get(C&: getLLVMContext(), NumBits: SVEBitsPerBlock / NumOpnds);
4261
4262 SmallVector<llvm::Value *, 16> VecOps;
4263 for (unsigned I = 0; I < NumOpnds; ++I)
4264 VecOps.push_back(Elt: Builder.CreateZExt(V: Ops[I], DestTy: EltTy));
4265 Value *Vec = BuildVector(Ops: VecOps);
4266
4267 llvm::Type *OverloadedTy = getSVEVectorForElementType(EltTy);
4268 Value *InsertSubVec = Builder.CreateInsertVector(
4269 DstType: OverloadedTy, SrcVec: PoisonValue::get(T: OverloadedTy), SubVec: Vec, Idx: uint64_t(0));
4270
4271 Function *F =
4272 CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_dupq_lane, Tys: OverloadedTy);
4273 Value *DupQLane =
4274 Builder.CreateCall(Callee: F, Args: {InsertSubVec, Builder.getInt64(C: 0)});
4275
4276 if (!IsBoolTy)
4277 return DupQLane;
4278
4279 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4280 Constant *Pred = ConstantInt::getTrue(Ty: getSVEPredType(TypeFlags));
4281
4282 // For svdupq_n_b* we need to add an additional 'cmpne' with '0'.
4283 F = CGM.getIntrinsic(IID: NumOpnds == 2 ? Intrinsic::aarch64_sve_cmpne
4284 : Intrinsic::aarch64_sve_cmpne_wide,
4285 Tys: OverloadedTy);
4286 Value *Call = Builder.CreateCall(
4287 Callee: F, Args: {Pred, DupQLane, EmitSVEDupX(Scalar: Builder.getInt64(C: 0))});
4288 return EmitSVEPredicateCast(Pred: Call, VTy: cast<llvm::ScalableVectorType>(Val: Ty));
4289 }
4290
4291 case SVE::BI__builtin_sve_svpfalse_b:
4292 return ConstantInt::getFalse(Ty);
4293
4294 case SVE::BI__builtin_sve_svpfalse_c:
4295 return Constant::getNullValue(Ty);
4296
4297 case SVE::BI__builtin_sve_svlen_bf16:
4298 case SVE::BI__builtin_sve_svlen_f16:
4299 case SVE::BI__builtin_sve_svlen_f32:
4300 case SVE::BI__builtin_sve_svlen_f64:
4301 case SVE::BI__builtin_sve_svlen_s8:
4302 case SVE::BI__builtin_sve_svlen_s16:
4303 case SVE::BI__builtin_sve_svlen_s32:
4304 case SVE::BI__builtin_sve_svlen_s64:
4305 case SVE::BI__builtin_sve_svlen_u8:
4306 case SVE::BI__builtin_sve_svlen_u16:
4307 case SVE::BI__builtin_sve_svlen_u32:
4308 case SVE::BI__builtin_sve_svlen_u64: {
4309 SVETypeFlags TF(Builtin->TypeModifier);
4310 return Builder.CreateElementCount(Ty, EC: getSVEType(TypeFlags: TF)->getElementCount());
4311 }
4312
4313 case SVE::BI__builtin_sve_svtbl2_u8:
4314 case SVE::BI__builtin_sve_svtbl2_s8:
4315 case SVE::BI__builtin_sve_svtbl2_u16:
4316 case SVE::BI__builtin_sve_svtbl2_s16:
4317 case SVE::BI__builtin_sve_svtbl2_u32:
4318 case SVE::BI__builtin_sve_svtbl2_s32:
4319 case SVE::BI__builtin_sve_svtbl2_u64:
4320 case SVE::BI__builtin_sve_svtbl2_s64:
4321 case SVE::BI__builtin_sve_svtbl2_f16:
4322 case SVE::BI__builtin_sve_svtbl2_bf16:
4323 case SVE::BI__builtin_sve_svtbl2_f32:
4324 case SVE::BI__builtin_sve_svtbl2_f64: {
4325 SVETypeFlags TF(Builtin->TypeModifier);
4326 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_sve_tbl2, Tys: getSVEType(TypeFlags: TF));
4327 return Builder.CreateCall(Callee: F, Args: Ops);
4328 }
4329
4330 case SVE::BI__builtin_sve_svset_neonq_s8:
4331 case SVE::BI__builtin_sve_svset_neonq_s16:
4332 case SVE::BI__builtin_sve_svset_neonq_s32:
4333 case SVE::BI__builtin_sve_svset_neonq_s64:
4334 case SVE::BI__builtin_sve_svset_neonq_u8:
4335 case SVE::BI__builtin_sve_svset_neonq_u16:
4336 case SVE::BI__builtin_sve_svset_neonq_u32:
4337 case SVE::BI__builtin_sve_svset_neonq_u64:
4338 case SVE::BI__builtin_sve_svset_neonq_f16:
4339 case SVE::BI__builtin_sve_svset_neonq_f32:
4340 case SVE::BI__builtin_sve_svset_neonq_f64:
4341 case SVE::BI__builtin_sve_svset_neonq_bf16:
4342 case SVE::BI__builtin_sve_svset_neonq_mf8: {
4343 return Builder.CreateInsertVector(DstType: Ty, SrcVec: Ops[0], SubVec: Ops[1], Idx: uint64_t(0));
4344 }
4345
4346 case SVE::BI__builtin_sve_svget_neonq_s8:
4347 case SVE::BI__builtin_sve_svget_neonq_s16:
4348 case SVE::BI__builtin_sve_svget_neonq_s32:
4349 case SVE::BI__builtin_sve_svget_neonq_s64:
4350 case SVE::BI__builtin_sve_svget_neonq_u8:
4351 case SVE::BI__builtin_sve_svget_neonq_u16:
4352 case SVE::BI__builtin_sve_svget_neonq_u32:
4353 case SVE::BI__builtin_sve_svget_neonq_u64:
4354 case SVE::BI__builtin_sve_svget_neonq_f16:
4355 case SVE::BI__builtin_sve_svget_neonq_f32:
4356 case SVE::BI__builtin_sve_svget_neonq_f64:
4357 case SVE::BI__builtin_sve_svget_neonq_bf16:
4358 case SVE::BI__builtin_sve_svget_neonq_mf8: {
4359 return Builder.CreateExtractVector(DstType: Ty, SrcVec: Ops[0], Idx: uint64_t(0));
4360 }
4361
4362 case SVE::BI__builtin_sve_svdup_neonq_s8:
4363 case SVE::BI__builtin_sve_svdup_neonq_s16:
4364 case SVE::BI__builtin_sve_svdup_neonq_s32:
4365 case SVE::BI__builtin_sve_svdup_neonq_s64:
4366 case SVE::BI__builtin_sve_svdup_neonq_u8:
4367 case SVE::BI__builtin_sve_svdup_neonq_u16:
4368 case SVE::BI__builtin_sve_svdup_neonq_u32:
4369 case SVE::BI__builtin_sve_svdup_neonq_u64:
4370 case SVE::BI__builtin_sve_svdup_neonq_f16:
4371 case SVE::BI__builtin_sve_svdup_neonq_f32:
4372 case SVE::BI__builtin_sve_svdup_neonq_f64:
4373 case SVE::BI__builtin_sve_svdup_neonq_bf16:
4374 case SVE::BI__builtin_sve_svdup_neonq_mf8: {
4375 Value *Insert = Builder.CreateInsertVector(DstType: Ty, SrcVec: PoisonValue::get(T: Ty), SubVec: Ops[0],
4376 Idx: uint64_t(0));
4377 return Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_dupq_lane, OverloadTypes: {Ty},
4378 Args: {Insert, Builder.getInt64(C: 0)});
4379 }
4380 }
4381
4382 /// Should not happen
4383 return nullptr;
4384}
4385
4386static void swapCommutativeSMEOperands(unsigned BuiltinID,
4387 SmallVectorImpl<Value *> &Ops) {
4388 unsigned MultiVec;
4389 switch (BuiltinID) {
4390 default:
4391 return;
4392 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
4393 MultiVec = 1;
4394 break;
4395 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
4396 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
4397 MultiVec = 2;
4398 break;
4399 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
4400 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
4401 MultiVec = 4;
4402 break;
4403 }
4404
4405 if (MultiVec > 0)
4406 for (unsigned I = 0; I < MultiVec; ++I)
4407 std::swap(a&: Ops[I + 1], b&: Ops[I + 1 + MultiVec]);
4408}
4409
4410Value *CodeGenFunction::EmitAArch64SMEBuiltinExpr(unsigned BuiltinID,
4411 const CallExpr *E) {
4412 auto *Builtin =
4413 findARMVectorIntrinsicInMap(IntrinsicMap: ArrayRef(AArch64SMEIntrinsicMap), BuiltinID,
4414 MapProvenSorted&: AArch64SMEIntrinsicsProvenSorted);
4415
4416 llvm::SmallVector<Value *, 4> Ops;
4417 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4418 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
4419
4420 if (TypeFlags.isLoad() || TypeFlags.isStore())
4421 return EmitSMELd1St1(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4422 if (TypeFlags.isReadZA() || TypeFlags.isWriteZA())
4423 return EmitSMEReadWrite(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4424 if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
4425 BuiltinID == SME::BI__builtin_sme_svzero_za)
4426 return EmitSMEZero(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4427 if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
4428 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
4429 BuiltinID == SME::BI__builtin_sme_svldr_za ||
4430 BuiltinID == SME::BI__builtin_sme_svstr_za)
4431 return EmitSMELdrStr(TypeFlags, Ops, IntID: Builtin->LLVMIntrinsic);
4432
4433 // Emit set FPMR for intrinsics that require it
4434 if (TypeFlags.setsFPMR())
4435 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_set_fpmr),
4436 Args: Ops.pop_back_val());
4437 // Handle builtins which require their multi-vector operands to be swapped
4438 swapCommutativeSMEOperands(BuiltinID, Ops);
4439
4440 auto isCntsBuiltin = [&]() {
4441 switch (BuiltinID) {
4442 default:
4443 return 0;
4444 case SME::BI__builtin_sme_svcntsb:
4445 return 8;
4446 case SME::BI__builtin_sme_svcntsh:
4447 return 4;
4448 case SME::BI__builtin_sme_svcntsw:
4449 return 2;
4450 }
4451 };
4452
4453 if (auto Mul = isCntsBuiltin()) {
4454 llvm::Value *Cntd =
4455 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_sme_cntsd));
4456 return Builder.CreateMul(LHS: Cntd, RHS: llvm::ConstantInt::get(Ty: Int64Ty, V: Mul),
4457 Name: "mulsvl", /* HasNUW */ true, /* HasNSW */ true);
4458 }
4459
4460 // Should not happen!
4461 if (Builtin->LLVMIntrinsic == 0)
4462 return nullptr;
4463
4464 // Predicates must match the main datatype.
4465 for (Value *&Op : Ops)
4466 if (auto PredTy = dyn_cast<llvm::VectorType>(Val: Op->getType()))
4467 if (PredTy->getElementType()->isIntegerTy(BitWidth: 1))
4468 Op = EmitSVEPredicateCast(Pred: Op, VTy: getSVEType(TypeFlags));
4469
4470 if (BuiltinID == SME::BI__builtin_sme_svldr_zt ||
4471 BuiltinID == SME::BI__builtin_sme_svstr_zt) {
4472 Function *F = CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic, Tys: Ops[1]->getType());
4473 return Builder.CreateCall(Callee: F, Args: Ops);
4474 }
4475
4476 Function *F =
4477 TypeFlags.isOverloadNone()
4478 ? CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic)
4479 : CGM.getIntrinsic(IID: Builtin->LLVMIntrinsic, Tys: {getSVEType(TypeFlags)});
4480
4481 return Builder.CreateCall(Callee: F, Args: Ops);
4482}
4483
4484/// Helper for the read/write/add/inc X18 builtins: read the X18 register and
4485/// return it as an i8 pointer.
4486Value *readX18AsPtr(CodeGenFunction &CGF) {
4487 LLVMContext &Context = CGF.CGM.getLLVMContext();
4488 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Str: "x18")};
4489 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
4490 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
4491 llvm::Function *F =
4492 CGF.CGM.getIntrinsic(IID: Intrinsic::read_register, Tys: {CGF.Int64Ty});
4493 llvm::Value *X18 = CGF.Builder.CreateCall(Callee: F, Args: Metadata);
4494 return CGF.Builder.CreateIntToPtr(V: X18, DestTy: CGF.Int8PtrTy);
4495}
4496
4497Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
4498 const CallExpr *E,
4499 llvm::Triple::ArchType Arch) {
4500 if (BuiltinID >= clang::AArch64::FirstSVEBuiltin &&
4501 BuiltinID <= clang::AArch64::LastSVEBuiltin)
4502 return EmitAArch64SVEBuiltinExpr(BuiltinID, E);
4503
4504 if (BuiltinID >= clang::AArch64::FirstSMEBuiltin &&
4505 BuiltinID <= clang::AArch64::LastSMEBuiltin)
4506 return EmitAArch64SMEBuiltinExpr(BuiltinID, E);
4507
4508 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
4509 return EmitAArch64CpuSupports(E);
4510
4511 unsigned HintID = static_cast<unsigned>(-1);
4512 switch (BuiltinID) {
4513 default: break;
4514 case clang::AArch64::BI__builtin_arm_nop:
4515 HintID = 0;
4516 break;
4517 case clang::AArch64::BI__builtin_arm_yield:
4518 case clang::AArch64::BI__yield:
4519 HintID = 1;
4520 break;
4521 case clang::AArch64::BI__builtin_arm_wfe:
4522 case clang::AArch64::BI__wfe:
4523 HintID = 2;
4524 break;
4525 case clang::AArch64::BI__builtin_arm_wfi:
4526 case clang::AArch64::BI__wfi:
4527 HintID = 3;
4528 break;
4529 case clang::AArch64::BI__builtin_arm_sev:
4530 case clang::AArch64::BI__sev:
4531 HintID = 4;
4532 break;
4533 case clang::AArch64::BI__builtin_arm_sevl:
4534 case clang::AArch64::BI__sevl:
4535 HintID = 5;
4536 break;
4537 }
4538
4539 if (HintID != static_cast<unsigned>(-1)) {
4540 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_hint);
4541 return Builder.CreateCall(Callee: F, Args: llvm::ConstantInt::get(Ty: Int32Ty, V: HintID));
4542 }
4543
4544 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
4545 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_break);
4546 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4547 return Builder.CreateCall(Callee: F, Args: Builder.CreateZExt(V: Arg, DestTy: CGM.Int32Ty));
4548 }
4549
4550 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
4551 // Create call to __arm_sme_state and store the results to the two pointers.
4552 CallInst *CI = EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(
4553 Ty: llvm::FunctionType::get(Result: StructType::get(elt1: CGM.Int64Ty, elts: CGM.Int64Ty), Params: {},
4554 isVarArg: false),
4555 Name: "__arm_sme_state"));
4556 auto Attrs = AttributeList().addFnAttribute(C&: getLLVMContext(),
4557 Kind: "aarch64_pstate_sm_compatible");
4558 CI->setAttributes(Attrs);
4559 CI->setCallingConv(
4560 llvm::CallingConv::
4561 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
4562 Builder.CreateStore(Val: Builder.CreateExtractValue(Agg: CI, Idxs: 0),
4563 Addr: EmitPointerWithAlignment(Addr: E->getArg(Arg: 0)));
4564 return Builder.CreateStore(Val: Builder.CreateExtractValue(Agg: CI, Idxs: 1),
4565 Addr: EmitPointerWithAlignment(Addr: E->getArg(Arg: 1)));
4566 }
4567
4568 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
4569 assert((getContext().getTypeSize(E->getType()) == 32) &&
4570 "rbit of unusual size!");
4571 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4572 return Builder.CreateCall(
4573 Callee: CGM.getIntrinsic(IID: Intrinsic::bitreverse, Tys: Arg->getType()), Args: Arg, Name: "rbit");
4574 }
4575 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
4576 assert((getContext().getTypeSize(E->getType()) == 64) &&
4577 "rbit of unusual size!");
4578 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4579 return Builder.CreateCall(
4580 Callee: CGM.getIntrinsic(IID: Intrinsic::bitreverse, Tys: Arg->getType()), Args: Arg, Name: "rbit");
4581 }
4582
4583 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
4584 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
4585 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4586 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: Arg->getType());
4587 Value *Res = Builder.CreateCall(Callee: F, Args: {Arg, Builder.getInt1(V: false)});
4588 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
4589 Res = Builder.CreateTrunc(V: Res, DestTy: Builder.getInt32Ty());
4590 return Res;
4591 }
4592
4593 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
4594 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4595 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_cls), Args: Arg,
4596 Name: "cls");
4597 }
4598 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
4599 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4600 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_cls64), Args: Arg,
4601 Name: "cls");
4602 }
4603
4604 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
4605 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
4606 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4607 llvm::Type *Ty = Arg->getType();
4608 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint32z, Tys: Ty),
4609 Args: Arg, Name: "frint32z");
4610 }
4611
4612 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
4613 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
4614 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4615 llvm::Type *Ty = Arg->getType();
4616 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint64z, Tys: Ty),
4617 Args: Arg, Name: "frint64z");
4618 }
4619
4620 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
4621 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
4622 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4623 llvm::Type *Ty = Arg->getType();
4624 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint32x, Tys: Ty),
4625 Args: Arg, Name: "frint32x");
4626 }
4627
4628 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
4629 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
4630 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4631 llvm::Type *Ty = Arg->getType();
4632 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_frint64x, Tys: Ty),
4633 Args: Arg, Name: "frint64x");
4634 }
4635
4636 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
4637 assert((getContext().getTypeSize(E->getType()) == 32) &&
4638 "__jcvt of unusual size!");
4639 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4640 return Builder.CreateCall(
4641 Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_fjcvtzs), Args: Arg);
4642 }
4643
4644 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
4645 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
4646 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
4647 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
4648 llvm::Value *MemAddr = EmitScalarExpr(E: E->getArg(Arg: 0));
4649 llvm::Value *ValPtr = EmitScalarExpr(E: E->getArg(Arg: 1));
4650
4651 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
4652 // Load from the address via an LLVM intrinsic, receiving a
4653 // tuple of 8 i64 words, and store each one to ValPtr.
4654 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_ld64b);
4655 llvm::Value *Val = Builder.CreateCall(Callee: F, Args: MemAddr);
4656 llvm::Value *ToRet;
4657 for (size_t i = 0; i < 8; i++) {
4658 llvm::Value *ValOffsetPtr =
4659 Builder.CreateGEP(Ty: Int64Ty, Ptr: ValPtr, IdxList: Builder.getInt32(C: i));
4660 Address Addr =
4661 Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(Quantity: 8));
4662 ToRet = Builder.CreateStore(Val: Builder.CreateExtractValue(Agg: Val, Idxs: i), Addr);
4663 }
4664 return ToRet;
4665 }
4666
4667 // Load 8 i64 words from ValPtr, and store them to the address
4668 // via an LLVM intrinsic.
4669 SmallVector<llvm::Value *, 9> Args;
4670 Args.push_back(Elt: MemAddr);
4671 for (size_t i = 0; i < 8; i++) {
4672 llvm::Value *ValOffsetPtr =
4673 Builder.CreateGEP(Ty: Int64Ty, Ptr: ValPtr, IdxList: Builder.getInt32(C: i));
4674 Address Addr = Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(Quantity: 8));
4675 Args.push_back(Elt: Builder.CreateLoad(Addr));
4676 }
4677
4678 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
4679 ? Intrinsic::aarch64_st64b
4680 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
4681 ? Intrinsic::aarch64_st64bv
4682 : Intrinsic::aarch64_st64bv0);
4683 Function *F = CGM.getIntrinsic(IID: Intr);
4684 return Builder.CreateCall(Callee: F, Args);
4685 }
4686
4687 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
4688 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
4689
4690 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
4691 ? Intrinsic::aarch64_rndr
4692 : Intrinsic::aarch64_rndrrs);
4693 Function *F = CGM.getIntrinsic(IID: Intr);
4694 llvm::Value *Val = Builder.CreateCall(Callee: F);
4695 Value *RandomValue = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
4696 Value *Status = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
4697
4698 Address MemAddress = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4699 Builder.CreateStore(Val: RandomValue, Addr: MemAddress);
4700 Status = Builder.CreateZExt(V: Status, DestTy: Int32Ty);
4701 return Status;
4702 }
4703
4704 if (BuiltinID == clang::AArch64::BI__clear_cache) {
4705 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
4706 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
4707 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache, Tys: {CGM.DefaultPtrTy});
4708 return Builder.CreateCall(Callee: F, Args: {Begin, End});
4709 }
4710
4711 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4712 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
4713 getContext().getTypeSize(T: E->getType()) == 128) {
4714 Function *F =
4715 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4716 ? Intrinsic::aarch64_ldaxp
4717 : Intrinsic::aarch64_ldxp);
4718
4719 Value *LdPtr = EmitScalarExpr(E: E->getArg(Arg: 0));
4720 Value *Val = Builder.CreateCall(Callee: F, Args: LdPtr, Name: "ldxp");
4721
4722 Value *Val0 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
4723 Value *Val1 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
4724 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 128);
4725 Val0 = Builder.CreateZExt(V: Val0, DestTy: Int128Ty);
4726 Val1 = Builder.CreateZExt(V: Val1, DestTy: Int128Ty);
4727
4728 Value *ShiftCst = llvm::ConstantInt::get(Ty: Int128Ty, V: 64);
4729 Val = Builder.CreateShl(LHS: Val0, RHS: ShiftCst, Name: "shl", HasNUW: true /* nuw */);
4730 Val = Builder.CreateOr(LHS: Val, RHS: Val1);
4731 return Builder.CreateBitCast(V: Val, DestTy: ConvertType(T: E->getType()));
4732 } else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4733 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
4734 Value *LoadAddr = EmitScalarExpr(E: E->getArg(Arg: 0));
4735
4736 QualType Ty = E->getType();
4737 llvm::Type *RealResTy = ConvertType(T: Ty);
4738 llvm::Type *IntTy =
4739 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
4740
4741 Function *F =
4742 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4743 ? Intrinsic::aarch64_ldaxr
4744 : Intrinsic::aarch64_ldxr,
4745 Tys: DefaultPtrTy);
4746 CallInst *Val = Builder.CreateCall(Callee: F, Args: LoadAddr, Name: "ldxr");
4747 Val->addParamAttr(
4748 ArgNo: 0, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: IntTy));
4749
4750 if (RealResTy->isPointerTy())
4751 return Builder.CreateIntToPtr(V: Val, DestTy: RealResTy);
4752
4753 llvm::Type *IntResTy = llvm::IntegerType::get(
4754 C&: getLLVMContext(), NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: RealResTy));
4755 return Builder.CreateBitCast(V: Builder.CreateTruncOrBitCast(V: Val, DestTy: IntResTy),
4756 DestTy: RealResTy);
4757 }
4758
4759 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4760 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
4761 getContext().getTypeSize(T: E->getArg(Arg: 0)->getType()) == 128) {
4762 Function *F =
4763 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4764 ? Intrinsic::aarch64_stlxp
4765 : Intrinsic::aarch64_stxp);
4766 llvm::Type *STy = llvm::StructType::get(elt1: Int64Ty, elts: Int64Ty);
4767
4768 Address Tmp = CreateMemTempWithoutCast(T: E->getArg(Arg: 0)->getType());
4769 EmitAnyExprToMem(E: E->getArg(Arg: 0), Location: Tmp, Quals: Qualifiers(), /*init*/ IsInitializer: true);
4770
4771 Tmp = Tmp.withElementType(ElemTy: STy);
4772 llvm::Value *Val = Builder.CreateLoad(Addr: Tmp);
4773
4774 Value *Arg0 = Builder.CreateExtractValue(Agg: Val, Idxs: 0);
4775 Value *Arg1 = Builder.CreateExtractValue(Agg: Val, Idxs: 1);
4776 Value *StPtr = EmitScalarExpr(E: E->getArg(Arg: 1));
4777 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1, StPtr}, Name: "stxp");
4778 }
4779
4780 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4781 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
4782 Value *StoreVal = EmitScalarExpr(E: E->getArg(Arg: 0));
4783 Value *StoreAddr = EmitScalarExpr(E: E->getArg(Arg: 1));
4784
4785 QualType Ty = E->getArg(Arg: 0)->getType();
4786 llvm::Type *StoreTy =
4787 llvm::IntegerType::get(C&: getLLVMContext(), NumBits: getContext().getTypeSize(T: Ty));
4788
4789 if (StoreVal->getType()->isPointerTy())
4790 StoreVal = Builder.CreatePtrToInt(V: StoreVal, DestTy: Int64Ty);
4791 else {
4792 llvm::Type *IntTy = llvm::IntegerType::get(
4793 C&: getLLVMContext(),
4794 NumBits: CGM.getDataLayout().getTypeSizeInBits(Ty: StoreVal->getType()));
4795 StoreVal = Builder.CreateBitCast(V: StoreVal, DestTy: IntTy);
4796 StoreVal = Builder.CreateZExtOrBitCast(V: StoreVal, DestTy: Int64Ty);
4797 }
4798
4799 Function *F =
4800 CGM.getIntrinsic(IID: BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4801 ? Intrinsic::aarch64_stlxr
4802 : Intrinsic::aarch64_stxr,
4803 Tys: StoreAddr->getType());
4804 CallInst *CI = Builder.CreateCall(Callee: F, Args: {StoreVal, StoreAddr}, Name: "stxr");
4805 CI->addParamAttr(
4806 ArgNo: 1, Attr: Attribute::get(Context&: getLLVMContext(), Kind: Attribute::ElementType, Ty: StoreTy));
4807 return CI;
4808 }
4809
4810 if (BuiltinID == clang::AArch64::BI__getReg ||
4811 BuiltinID == clang::AArch64::BI__setReg) {
4812 Expr::EvalResult Result;
4813 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4814 llvm_unreachable("Sema will ensure that the parameter is constant");
4815
4816 llvm::APSInt Value = Result.Val.getInt();
4817 LLVMContext &Context = CGM.getLLVMContext();
4818 std::string Reg = Value == 31 ? "sp" : "x" + toString(I: Value, Radix: 10);
4819
4820 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Str: Reg)};
4821 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
4822 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
4823
4824 CallInst *CI;
4825 if (BuiltinID == clang::AArch64::BI__getReg) {
4826 llvm::Function *F =
4827 CGM.getIntrinsic(IID: Intrinsic::read_volatile_register, Tys: {Int64Ty});
4828 CI = Builder.CreateCall(Callee: F, Args: Metadata);
4829 } else {
4830 llvm::Function *F =
4831 CGM.getIntrinsic(IID: Intrinsic::write_volatile_register, Tys: {Int64Ty});
4832 CI = Builder.CreateCall(Callee: F, Args: {Metadata, EmitScalarExpr(E: E->getArg(Arg: 1))});
4833 }
4834 return CI;
4835 }
4836
4837 if (BuiltinID == clang::AArch64::BI__getRegFp ||
4838 BuiltinID == clang::AArch64::BI__setRegFp) {
4839 Expr::EvalResult Result;
4840 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4841 llvm_unreachable("Sema will ensure that the parameter is constant");
4842
4843 llvm::APSInt Value = Result.Val.getInt();
4844 LLVMContext &Context = CGM.getLLVMContext();
4845 std::string Reg = "d" + toString(I: Value, Radix: 10);
4846
4847 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Str: Reg)};
4848 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
4849 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
4850
4851 llvm::Value *Ret;
4852 if (BuiltinID == clang::AArch64::BI__getRegFp) {
4853 llvm::Function *F =
4854 CGM.getIntrinsic(IID: Intrinsic::read_volatile_register, Tys: {Int64Ty});
4855 llvm::Value *Bits = Builder.CreateCall(Callee: F, Args: Metadata);
4856 Ret = Builder.CreateBitCast(V: Bits, DestTy: llvm::Type::getDoubleTy(C&: Context));
4857 } else {
4858 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
4859 llvm::Value *Bits = Builder.CreateBitCast(V: Val, DestTy: Int64Ty);
4860 llvm::Function *F =
4861 CGM.getIntrinsic(IID: Intrinsic::write_volatile_register, Tys: {Int64Ty});
4862 Ret = Builder.CreateCall(Callee: F, Args: {Metadata, Bits});
4863 }
4864 return Ret;
4865 }
4866
4867 if (BuiltinID == clang::AArch64::BI__break) {
4868 Expr::EvalResult Result;
4869 if (!E->getArg(Arg: 0)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4870 llvm_unreachable("Sema will ensure that the parameter is constant");
4871
4872 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_break);
4873 return Builder.CreateCall(Callee: F, Args: {EmitScalarExpr(E: E->getArg(Arg: 0))});
4874 }
4875
4876 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
4877 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_clrex);
4878 return Builder.CreateCall(Callee: F);
4879 }
4880
4881 // CRC32
4882 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
4883 switch (BuiltinID) {
4884 case clang::AArch64::BI__builtin_arm_crc32b:
4885 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
4886 case clang::AArch64::BI__builtin_arm_crc32cb:
4887 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
4888 case clang::AArch64::BI__builtin_arm_crc32h:
4889 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
4890 case clang::AArch64::BI__builtin_arm_crc32ch:
4891 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
4892 case clang::AArch64::BI__builtin_arm_crc32w:
4893 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
4894 case clang::AArch64::BI__builtin_arm_crc32cw:
4895 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
4896 case clang::AArch64::BI__builtin_arm_crc32d:
4897 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
4898 case clang::AArch64::BI__builtin_arm_crc32cd:
4899 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
4900 }
4901
4902 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4903 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4904 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4905 Function *F = CGM.getIntrinsic(IID: CRCIntrinsicID);
4906
4907 llvm::Type *DataTy = F->getFunctionType()->getParamType(i: 1);
4908 Arg1 = Builder.CreateZExtOrBitCast(V: Arg1, DestTy: DataTy);
4909
4910 return Builder.CreateCall(Callee: F, Args: {Arg0, Arg1});
4911 }
4912
4913 // Memory Operations (MOPS)
4914 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
4915 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
4916 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
4917 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
4918 Val = Builder.CreateTrunc(V: Val, DestTy: Int8Ty);
4919 Size = Builder.CreateIntCast(V: Size, DestTy: Int64Ty, isSigned: false);
4920 return Builder.CreateCall(
4921 Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_mops_memset_tag), Args: {Dst, Val, Size});
4922 }
4923
4924 if (BuiltinID == AArch64::BI__builtin_arm_range_prefetch ||
4925 BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
4926 return EmitRangePrefetchBuiltin(CGF&: *this, BuiltinID, E);
4927
4928 switch (BuiltinID) {
4929 default:
4930 break;
4931 case AArch64::BI__builtin_arm_atomic_store_with_hint:
4932 return EmitAtomicStoreWithHintBuiltin(CGF&: *this, BuiltinID, E);
4933 case AArch64::BI__builtin_arm_atomic_fetch_add_with_hint:
4934 return EmitAtomicFetchWithHintBuiltin(CGF&: *this, E, Op: AtomicRMWInst::Add);
4935 case AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint:
4936 return EmitAtomicFetchWithHintBuiltin(CGF&: *this, E, Op: AtomicRMWInst::Sub);
4937 case AArch64::BI__builtin_arm_atomic_fetch_and_with_hint:
4938 return EmitAtomicFetchWithHintBuiltin(CGF&: *this, E, Op: AtomicRMWInst::And);
4939 case AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint:
4940 return EmitAtomicFetchWithHintBuiltin(CGF&: *this, E, Op: AtomicRMWInst::Xor);
4941 case AArch64::BI__builtin_arm_atomic_fetch_or_with_hint:
4942 return EmitAtomicFetchWithHintBuiltin(CGF&: *this, E, Op: AtomicRMWInst::Or);
4943 }
4944
4945 // Memory Tagging Extensions (MTE) Intrinsics
4946 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
4947 switch (BuiltinID) {
4948 case clang::AArch64::BI__builtin_arm_irg:
4949 MTEIntrinsicID = Intrinsic::aarch64_irg; break;
4950 case clang::AArch64::BI__builtin_arm_addg:
4951 MTEIntrinsicID = Intrinsic::aarch64_addg; break;
4952 case clang::AArch64::BI__builtin_arm_gmi:
4953 MTEIntrinsicID = Intrinsic::aarch64_gmi; break;
4954 case clang::AArch64::BI__builtin_arm_ldg:
4955 MTEIntrinsicID = Intrinsic::aarch64_ldg; break;
4956 case clang::AArch64::BI__builtin_arm_stg:
4957 MTEIntrinsicID = Intrinsic::aarch64_stg; break;
4958 case clang::AArch64::BI__builtin_arm_subp:
4959 MTEIntrinsicID = Intrinsic::aarch64_subp; break;
4960 }
4961
4962 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_lscale) {
4963 Value *FPM = EmitScalarExpr(E: E->getArg(Arg: 0));
4964 Value *Scale = EmitScalarExpr(E: E->getArg(Arg: 1));
4965 Scale = Builder.CreateAnd(LHS: Scale, RHS: Builder.getInt64(C: 0x7f));
4966
4967 Value *MaskedFPM = Builder.CreateAnd(LHS: FPM, RHS: Builder.getInt64(C: ~0x7f0000ULL));
4968 Value *ShiftedScale = Builder.CreateShl(LHS: Scale, RHS: Builder.getInt64(C: 16));
4969
4970 return Builder.CreateOr(LHS: MaskedFPM, RHS: ShiftedScale);
4971 }
4972 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_nscale) {
4973 Value *FPM = EmitScalarExpr(E: E->getArg(Arg: 0));
4974 Value *Scale = EmitScalarExpr(E: E->getArg(Arg: 1));
4975 Scale = Builder.CreateAnd(LHS: Scale, RHS: Builder.getInt64(C: 0xff));
4976
4977 Value *MaskedFPM = Builder.CreateAnd(LHS: FPM, RHS: Builder.getInt64(C: ~0xff000000ULL));
4978 Value *ShiftedScale = Builder.CreateShl(LHS: Scale, RHS: Builder.getInt64(C: 24));
4979
4980 return Builder.CreateOr(LHS: MaskedFPM, RHS: ShiftedScale);
4981 }
4982 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_lscale2) {
4983 Value *FPM = EmitScalarExpr(E: E->getArg(Arg: 0));
4984 Value *Scale = EmitScalarExpr(E: E->getArg(Arg: 1));
4985 Scale = Builder.CreateAnd(LHS: Scale, RHS: Builder.getInt64(C: 0x3f));
4986
4987 Value *LowFPM = Builder.CreateAnd(LHS: FPM, RHS: Builder.getInt64(C: ~0x3f00000000ULL));
4988 Value *ShiftedScale = Builder.CreateShl(LHS: Scale, RHS: Builder.getInt64(C: 32));
4989
4990 return Builder.CreateOr(LHS: LowFPM, RHS: ShiftedScale);
4991 }
4992
4993 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
4994 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
4995 Value *Pointer = EmitScalarExpr(E: E->getArg(Arg: 0));
4996 Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 1));
4997 assert(Mask->getType()->getScalarSizeInBits() == 64 &&
4998 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
4999 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
5000 Args: {Pointer, Mask});
5001 }
5002 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
5003 Value *Pointer = EmitScalarExpr(E: E->getArg(Arg: 0));
5004 Value *TagOffset = EmitScalarExpr(E: E->getArg(Arg: 1));
5005
5006 TagOffset = Builder.CreateZExt(V: TagOffset, DestTy: Int64Ty);
5007 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
5008 Args: {Pointer, TagOffset});
5009 }
5010 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
5011 Value *Pointer = EmitScalarExpr(E: E->getArg(Arg: 0));
5012 Value *ExcludedMask = EmitScalarExpr(E: E->getArg(Arg: 1));
5013 assert(ExcludedMask->getType()->getScalarSizeInBits() == 64 &&
5014 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
5015 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
5016 Args: {Pointer, ExcludedMask});
5017 }
5018 // Although it is possible to supply a different return
5019 // address (first arg) to this intrinsic, for now we set
5020 // return address same as input address.
5021 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
5022 Value *TagAddress = EmitScalarExpr(E: E->getArg(Arg: 0));
5023 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
5024 Args: {TagAddress, TagAddress});
5025 }
5026 // Although it is possible to supply a different tag (to set)
5027 // to this intrinsic (as first arg), for now we supply
5028 // the tag that is in input address arg (common use case).
5029 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
5030 Value *TagAddress = EmitScalarExpr(E: E->getArg(Arg: 0));
5031 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: MTEIntrinsicID),
5032 Args: {TagAddress, TagAddress});
5033 }
5034 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
5035 Value *PointerA = EmitScalarExpr(E: E->getArg(Arg: 0));
5036 Value *PointerB = EmitScalarExpr(E: E->getArg(Arg: 1));
5037 return Builder.CreateCall(
5038 Callee: CGM.getIntrinsic(IID: MTEIntrinsicID), Args: {PointerA, PointerB});
5039 }
5040 }
5041
5042 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5043 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
5044 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5045 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
5046 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
5047 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
5048 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
5049 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
5050
5051 SpecialRegisterAccessKind AccessKind = Write;
5052 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5053 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
5054 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5055 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
5056 AccessKind = VolatileRead;
5057
5058 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
5059 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
5060
5061 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5062 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
5063
5064 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5065 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
5066
5067 llvm::Type *ValueType;
5068 llvm::Type *RegisterType = Int64Ty;
5069 if (Is32Bit) {
5070 ValueType = Int32Ty;
5071 } else if (Is128Bit) {
5072 llvm::Type *Int128Ty =
5073 llvm::IntegerType::getInt128Ty(C&: CGM.getLLVMContext());
5074 ValueType = Int128Ty;
5075 RegisterType = Int128Ty;
5076 } else if (IsPointerBuiltin) {
5077 ValueType = VoidPtrTy;
5078 } else {
5079 ValueType = Int64Ty;
5080 };
5081
5082 return EmitSpecialRegisterBuiltin(CGF&: *this, E, RegisterType, ValueType,
5083 AccessKind);
5084 }
5085
5086 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
5087 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
5088 LLVMContext &Context = CGM.getLLVMContext();
5089
5090 unsigned SysReg =
5091 E->getArg(Arg: 0)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
5092
5093 std::string SysRegStr;
5094 llvm::raw_string_ostream(SysRegStr)
5095 << (0b10 | SysReg >> 14) << ":" << ((SysReg >> 11) & 7) << ":"
5096 << ((SysReg >> 7) & 15) << ":" << ((SysReg >> 3) & 15) << ":"
5097 << (SysReg & 7);
5098
5099 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, Str: SysRegStr) };
5100 llvm::MDNode *RegName = llvm::MDNode::get(Context, MDs: Ops);
5101 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, MD: RegName);
5102
5103 llvm::Type *RegisterType = Int64Ty;
5104 llvm::Type *Types[] = { RegisterType };
5105
5106 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
5107 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::read_register, Tys: Types);
5108
5109 return Builder.CreateCall(Callee: F, Args: Metadata);
5110 }
5111
5112 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::write_register, Tys: Types);
5113 llvm::Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 1));
5114 llvm::Value *Result = Builder.CreateCall(Callee: F, Args: {Metadata, ArgValue});
5115
5116 return Result;
5117 }
5118
5119 if (BuiltinID == clang::AArch64::BI__sys) {
5120 unsigned SysReg =
5121 E->getArg(Arg: 0)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
5122 const unsigned Op1 = SysReg >> 11;
5123 const unsigned CRn = (SysReg >> 7) & 0xf;
5124 const unsigned CRm = (SysReg >> 3) & 0xf;
5125 const unsigned Op2 = SysReg & 0x7;
5126
5127 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::aarch64_sys),
5128 Args: {Builder.getInt32(C: Op1), Builder.getInt32(C: CRn),
5129 Builder.getInt32(C: CRm), Builder.getInt32(C: Op2),
5130 EmitScalarExpr(E: E->getArg(Arg: 1))});
5131
5132 // Return 0 for convenience, even though MSVC returns some other undefined
5133 // value.
5134 return ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
5135 }
5136
5137 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
5138 llvm::Function *F =
5139 CGM.getIntrinsic(IID: Intrinsic::addressofreturnaddress, Tys: AllocaInt8PtrTy);
5140 return Builder.CreateCall(Callee: F);
5141 }
5142
5143 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
5144 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: AllocaInt8PtrTy);
5145 return Builder.CreateCall(Callee: F);
5146 }
5147
5148 if (BuiltinID == clang::AArch64::BI__mulh ||
5149 BuiltinID == clang::AArch64::BI__umulh) {
5150 llvm::Type *ResType = ConvertType(T: E->getType());
5151 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 128);
5152
5153 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
5154 Value *LHS =
5155 Builder.CreateIntCast(V: EmitScalarExpr(E: E->getArg(Arg: 0)), DestTy: Int128Ty, isSigned: IsSigned);
5156 Value *RHS =
5157 Builder.CreateIntCast(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: Int128Ty, isSigned: IsSigned);
5158
5159 Value *MulResult, *HigherBits;
5160 if (IsSigned) {
5161 MulResult = Builder.CreateNSWMul(LHS, RHS);
5162 HigherBits = Builder.CreateAShr(LHS: MulResult, RHS: 64);
5163 } else {
5164 MulResult = Builder.CreateNUWMul(LHS, RHS);
5165 HigherBits = Builder.CreateLShr(LHS: MulResult, RHS: 64);
5166 }
5167 HigherBits = Builder.CreateIntCast(V: HigherBits, DestTy: ResType, isSigned: IsSigned);
5168
5169 return HigherBits;
5170 }
5171
5172 if (BuiltinID == AArch64::BI__writex18byte ||
5173 BuiltinID == AArch64::BI__writex18word ||
5174 BuiltinID == AArch64::BI__writex18dword ||
5175 BuiltinID == AArch64::BI__writex18qword) {
5176 // Process the args first
5177 Value *OffsetArg = EmitScalarExpr(E: E->getArg(Arg: 0));
5178 Value *DataArg = EmitScalarExpr(E: E->getArg(Arg: 1));
5179
5180 // Read x18 as i8*
5181 llvm::Value *X18 = readX18AsPtr(CGF&: *this);
5182
5183 // Store val at x18 + offset
5184 Value *Offset = Builder.CreateZExt(V: OffsetArg, DestTy: Int64Ty);
5185 Value *Ptr = Builder.CreateGEP(Ty: Int8Ty, Ptr: X18, IdxList: Offset);
5186 StoreInst *Store =
5187 Builder.CreateAlignedStore(Val: DataArg, Addr: Ptr, Align: CharUnits::One());
5188 return Store;
5189 }
5190
5191 if (BuiltinID == AArch64::BI__readx18byte ||
5192 BuiltinID == AArch64::BI__readx18word ||
5193 BuiltinID == AArch64::BI__readx18dword ||
5194 BuiltinID == AArch64::BI__readx18qword) {
5195 // Process the args first
5196 Value *OffsetArg = EmitScalarExpr(E: E->getArg(Arg: 0));
5197
5198 // Read x18 as i8*
5199 llvm::Value *X18 = readX18AsPtr(CGF&: *this);
5200
5201 // Load x18 + offset
5202 Value *Offset = Builder.CreateZExt(V: OffsetArg, DestTy: Int64Ty);
5203 Value *Ptr = Builder.CreateGEP(Ty: Int8Ty, Ptr: X18, IdxList: Offset);
5204 llvm::Type *IntTy = ConvertType(T: E->getType());
5205 LoadInst *Load = Builder.CreateAlignedLoad(Ty: IntTy, Addr: Ptr, Align: CharUnits::One());
5206 return Load;
5207 }
5208
5209 if (BuiltinID == AArch64::BI__addx18byte ||
5210 BuiltinID == AArch64::BI__addx18word ||
5211 BuiltinID == AArch64::BI__addx18dword ||
5212 BuiltinID == AArch64::BI__addx18qword ||
5213 BuiltinID == AArch64::BI__incx18byte ||
5214 BuiltinID == AArch64::BI__incx18word ||
5215 BuiltinID == AArch64::BI__incx18dword ||
5216 BuiltinID == AArch64::BI__incx18qword) {
5217 llvm::Type *IntTy;
5218 bool isIncrement;
5219 switch (BuiltinID) {
5220 case AArch64::BI__incx18byte:
5221 IntTy = Int8Ty;
5222 isIncrement = true;
5223 break;
5224 case AArch64::BI__incx18word:
5225 IntTy = Int16Ty;
5226 isIncrement = true;
5227 break;
5228 case AArch64::BI__incx18dword:
5229 IntTy = Int32Ty;
5230 isIncrement = true;
5231 break;
5232 case AArch64::BI__incx18qword:
5233 IntTy = Int64Ty;
5234 isIncrement = true;
5235 break;
5236 default:
5237 IntTy = ConvertType(T: E->getArg(Arg: 1)->getType());
5238 isIncrement = false;
5239 break;
5240 }
5241 // Process the args first
5242 Value *OffsetArg = EmitScalarExpr(E: E->getArg(Arg: 0));
5243 Value *ValToAdd =
5244 isIncrement ? ConstantInt::get(Ty: IntTy, V: 1) : EmitScalarExpr(E: E->getArg(Arg: 1));
5245
5246 // Read x18 as i8*
5247 llvm::Value *X18 = readX18AsPtr(CGF&: *this);
5248
5249 // Load x18 + offset
5250 Value *Offset = Builder.CreateZExt(V: OffsetArg, DestTy: Int64Ty);
5251 Value *Ptr = Builder.CreateGEP(Ty: Int8Ty, Ptr: X18, IdxList: Offset);
5252 LoadInst *Load = Builder.CreateAlignedLoad(Ty: IntTy, Addr: Ptr, Align: CharUnits::One());
5253
5254 // Add values
5255 Value *AddResult = Builder.CreateAdd(LHS: Load, RHS: ValToAdd);
5256
5257 // Store val at x18 + offset
5258 StoreInst *Store =
5259 Builder.CreateAlignedStore(Val: AddResult, Addr: Ptr, Align: CharUnits::One());
5260 return Store;
5261 }
5262
5263 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
5264 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
5265 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
5266 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
5267 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
5268 llvm::Type *RetTy = ConvertType(T: E->getType());
5269 return Builder.CreateBitCast(V: Arg, DestTy: RetTy);
5270 }
5271
5272 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5273 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5274 BuiltinID == AArch64::BI_CountLeadingZeros ||
5275 BuiltinID == AArch64::BI_CountLeadingZeros64) {
5276 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
5277 llvm::Type *ArgType = Arg->getType();
5278
5279 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5280 BuiltinID == AArch64::BI_CountLeadingOnes64)
5281 Arg = Builder.CreateXor(LHS: Arg, RHS: Constant::getAllOnesValue(Ty: ArgType));
5282
5283 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
5284 Value *Result = Builder.CreateCall(Callee: F, Args: {Arg, Builder.getInt1(V: false)});
5285
5286 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5287 BuiltinID == AArch64::BI_CountLeadingZeros64)
5288 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5289 return Result;
5290 }
5291
5292 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
5293 BuiltinID == AArch64::BI_CountLeadingSigns64) {
5294 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
5295
5296 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
5297 ? CGM.getIntrinsic(IID: Intrinsic::aarch64_cls)
5298 : CGM.getIntrinsic(IID: Intrinsic::aarch64_cls64);
5299
5300 Value *Result = Builder.CreateCall(Callee: F, Args: Arg, Name: "cls");
5301 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
5302 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5303 return Result;
5304 }
5305
5306 if (BuiltinID == AArch64::BI_CountOneBits ||
5307 BuiltinID == AArch64::BI_CountOneBits64) {
5308 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
5309 llvm::Type *ArgType = ArgValue->getType();
5310 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
5311
5312 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
5313 if (BuiltinID == AArch64::BI_CountOneBits64)
5314 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5315 return Result;
5316 }
5317
5318 if (BuiltinID == AArch64::BI_CountTrailingZeros ||
5319 BuiltinID == AArch64::BI_CountTrailingZeros64) {
5320 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
5321 llvm::Type *ArgType = ArgValue->getType();
5322 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
5323
5324 // MSVC leaves 0 undefined; use false for predictable codegen
5325 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getInt1(V: false)});
5326 if (BuiltinID == AArch64::BI_CountTrailingZeros64)
5327 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
5328 return Result;
5329 }
5330
5331 if (BuiltinID == AArch64::BI__prefetch) {
5332 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5333 Value *RW = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
5334 Value *Locality = ConstantInt::get(Ty: Int32Ty, V: 3);
5335 Value *Data = llvm::ConstantInt::get(Ty: Int32Ty, V: 1);
5336 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
5337 return Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, Data});
5338 }
5339
5340 if (BuiltinID == AArch64::BI__prefetch2) {
5341 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5342 llvm::APSInt PrfOp = E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: CGM.getContext());
5343 // Decode 5-bit PRFM encoding: bits[4:3]=type, bits[2:1]=target,
5344 // bit[0]=policy
5345 // type: PLD=0(load), PLI=1(instr), PST=2(store)
5346 // target: L1=0, L2=1, L3=2
5347 // policy: KEEP=0, STRM=1
5348 uint64_t Op = PrfOp.getZExtValue();
5349 uint64_t Type = (Op >> 3) & 0x3;
5350 uint64_t Target = (Op >> 1) & 0x3;
5351 uint64_t Policy = Op & 0x1;
5352 Value *RW = Builder.getInt32(C: Type == 2 ? 1 : 0);
5353 Value *Local = Builder.getInt32(C: Target);
5354 Value *IsStream = Builder.getInt32(C: Policy);
5355 Value *IsData = Builder.getInt32(C: Type == 1 ? 0 : 1);
5356 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_prefetch);
5357 return Builder.CreateCall(Callee: F, Args: {Address, RW, Local, IsStream, IsData});
5358 }
5359
5360 if (BuiltinID == AArch64::BI__hlt) {
5361 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_hlt);
5362 Builder.CreateCall(Callee: F, Args: {EmitScalarExpr(E: E->getArg(Arg: 0))});
5363
5364 // FIXME: MSVC documents __hlt as taking further arguments in X0-X3 and
5365 // returning the value in X0, like __hvc/__svc below. This ignores the
5366 // extra arguments and returns 0.
5367 return ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
5368 }
5369
5370 if (BuiltinID == AArch64::BI__hvc || BuiltinID == AArch64::BI__svc) {
5371 unsigned IID = BuiltinID == AArch64::BI__svc ? Intrinsic::aarch64_svc
5372 : Intrinsic::aarch64_hvc;
5373 // The first argument is the instruction immediate; it must be a constant
5374 // (ImmArg on the intrinsic, encoded in the instruction). The remaining
5375 // arguments (at most four, enforced by Sema) are passed in X0-X3, widened
5376 // to 64 bits. The intrinsic takes exactly four register operands, so any
5377 // unused trailing ones are passed as poison and dropped during lowering.
5378 SmallVector<Value *, 5> Args{Builder.getInt32(
5379 C: GetIntegerConstantValue<uint32_t>(E: E->getArg(Arg: 0), Context&: getContext()))};
5380 for (unsigned I = 1, N = E->getNumArgs(); I < N; ++I) {
5381 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: I));
5382 llvm::Type *ArgTy = Arg->getType();
5383 if (ArgTy->isPointerTy())
5384 Arg = Builder.CreatePtrToInt(V: Arg, DestTy: Int64Ty);
5385 else if (ArgTy->isFloatingPointTy())
5386 // Reinterpret the bits into the integer register, matching MSVC (e.g.
5387 // "fmov x0, d0" for a double).
5388 Arg = Builder.CreateZExtOrTrunc(
5389 V: Builder.CreateBitCast(
5390 V: Arg, DestTy: Builder.getIntNTy(N: ArgTy->getPrimitiveSizeInBits())),
5391 DestTy: Int64Ty);
5392 else
5393 Arg = Builder.CreateIntCast(
5394 V: Arg, DestTy: Int64Ty, isSigned: E->getArg(Arg: I)->getType()->isSignedIntegerType());
5395 Args.push_back(Elt: Arg);
5396 }
5397 while (Args.size() < 5)
5398 Args.push_back(Elt: llvm::PoisonValue::get(T: Int64Ty));
5399 Value *Call = Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args);
5400 // MSVC returns unsigned int, i.e. the low 32 bits of the X0 result.
5401 return Builder.CreateTrunc(V: Call, DestTy: Int32Ty);
5402 }
5403
5404 if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
5405 return Builder.CreateFPTrunc(
5406 V: Builder.CreateBitCast(V: EmitScalarExpr(E: E->getArg(Arg: 0)),
5407 DestTy: Builder.getFloatTy()),
5408 DestTy: Builder.getBFloatTy());
5409
5410 // Handle MSVC intrinsics before argument evaluation to prevent double
5411 // evaluation.
5412 if (std::optional<MSVCIntrin> MsvcIntId =
5413 translateAarch64ToMsvcIntrin(BuiltinID))
5414 return EmitMSVCBuiltinExpr(BuiltinID: *MsvcIntId, E);
5415
5416 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
5417 auto It = llvm::find_if(Range: NEONEquivalentIntrinsicMap, P: [BuiltinID](auto &P) {
5418 return P.first == BuiltinID;
5419 });
5420 if (It != end(arr: NEONEquivalentIntrinsicMap))
5421 BuiltinID = It->second;
5422
5423 // Check whether this is an SISD builtin.
5424 auto SISDMap = ArrayRef(AArch64SISDIntrinsicMap);
5425 const ARMNeonVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap(
5426 IntrinsicMap: SISDMap, BuiltinID, MapProvenSorted&: AArch64SISDIntrinsicsProvenSorted);
5427 bool IsSISD = (Builtin != nullptr);
5428
5429 // Find out if any arguments are required to be integer constant
5430 // expressions.
5431 unsigned ICEArguments = 0;
5432 ASTContext::GetBuiltinTypeError Error;
5433 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
5434 assert(Error == ASTContext::GE_None && "Should not codegen an error");
5435
5436 llvm::SmallVector<Value*, 4> Ops;
5437 Address PtrOp0 = Address::invalid();
5438 // Note the assumption that SISD intrinsics do not contain extra arguments.
5439 // TODO: Fold this into a single function call instead of, effectively, two
5440 // separate checks.
5441 bool HasExtraArg = !IsSISD && CodeGenUtils::hasExtraNeonArgument(BuiltinID);
5442 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
5443 for (unsigned i = 0, e = NumArgs; i != e; i++) {
5444 if (i == 0) {
5445 switch (BuiltinID) {
5446 case NEON::BI__builtin_neon_vld1_v:
5447 case NEON::BI__builtin_neon_vld1q_v:
5448 case NEON::BI__builtin_neon_vld1_dup_v:
5449 case NEON::BI__builtin_neon_vld1q_dup_v:
5450 case NEON::BI__builtin_neon_vld1_lane_v:
5451 case NEON::BI__builtin_neon_vld1q_lane_v:
5452 case NEON::BI__builtin_neon_vst1_v:
5453 case NEON::BI__builtin_neon_vst1q_v:
5454 case NEON::BI__builtin_neon_vst1_lane_v:
5455 case NEON::BI__builtin_neon_vst1q_lane_v:
5456 case NEON::BI__builtin_neon_vldap1_lane_s64:
5457 case NEON::BI__builtin_neon_vldap1q_lane_s64:
5458 case NEON::BI__builtin_neon_vstl1_lane_s64:
5459 case NEON::BI__builtin_neon_vstl1q_lane_s64:
5460 // Get the alignment for the argument in addition to the value;
5461 // we'll use it later.
5462 PtrOp0 = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5463 Ops.push_back(Elt: PtrOp0.emitRawPointer(CGF&: *this));
5464 continue;
5465 }
5466 }
5467 Ops.push_back(Elt: EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E));
5468 }
5469
5470 if (Builtin) {
5471 Value *Result = EmitCommonNeonSISDBuiltinExpr(CGF&: *this, SISDInfo: *Builtin, Ops, E);
5472 assert(Result && "SISD intrinsic should have been handled");
5473 return Result;
5474 }
5475
5476 const Expr *Arg = E->getArg(Arg: E->getNumArgs()-1);
5477 NeonTypeFlags Type(0);
5478 if (std::optional<llvm::APSInt> Result =
5479 Arg->getIntegerConstantExpr(Ctx: getContext()))
5480 // Determine the type of this overloaded NEON intrinsic.
5481 Type = NeonTypeFlags(Result->getZExtValue());
5482
5483 bool usgn = Type.isUnsigned();
5484 bool quad = Type.isQuad();
5485 unsigned Int;
5486
5487 // Not all intrinsics handled by the common case work for AArch64 yet, so only
5488 // defer to common code if it's been added to our special map.
5489 Builtin =
5490 findARMVectorIntrinsicInMap(IntrinsicMap: ArrayRef(AArch64SIMDIntrinsicMap), BuiltinID,
5491 MapProvenSorted&: AArch64SIMDIntrinsicsProvenSorted);
5492
5493 if (Builtin)
5494 return EmitCommonNeonBuiltinExpr(
5495 BuiltinID: Builtin->BuiltinID, LLVMIntrinsic: Builtin->LLVMIntrinsic, AltLLVMIntrinsic: Builtin->AltLLVMIntrinsic,
5496 NameHint: Builtin->NameHint, Modifier: Builtin->TypeModifier, E, Ops,
5497 /*never use addresses*/ PtrOp0: Address::invalid(), PtrOp1: Address::invalid(), Arch);
5498
5499 if (Value *V = EmitAArch64TblBuiltinExpr(CGF&: *this, BuiltinID, E, Ops, Arch))
5500 return V;
5501
5502 // Handle non-overloaded intrinsics first.
5503 switch (BuiltinID) {
5504 default: break;
5505 case NEON::BI__builtin_neon_vabsh_f16:
5506 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::fabs, Tys: HalfTy), Ops, name: "vabs");
5507 case NEON::BI__builtin_neon_vaddq_p128: {
5508 llvm::Type *Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags::Poly128);
5509 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
5510 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
5511 Ops[0] = Builder.CreateXor(LHS: Ops[0], RHS: Ops[1]);
5512 llvm::Type *Int128Ty = llvm::Type::getIntNTy(C&: getLLVMContext(), N: 128);
5513 return Builder.CreateBitCast(V: Ops[0], DestTy: Int128Ty);
5514 }
5515 case NEON::BI__builtin_neon_vldrq_p128: {
5516 llvm::Type *Int128Ty = llvm::Type::getIntNTy(C&: getLLVMContext(), N: 128);
5517 return Builder.CreateAlignedLoad(Ty: Int128Ty, Addr: Ops[0],
5518 Align: CharUnits::fromQuantity(Quantity: 16));
5519 }
5520 case NEON::BI__builtin_neon_vstrq_p128: {
5521 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
5522 }
5523 case NEON::BI__builtin_neon_vcvts_f32_u32:
5524 case NEON::BI__builtin_neon_vcvtd_f64_u64:
5525 usgn = true;
5526 [[fallthrough]];
5527 case NEON::BI__builtin_neon_vcvts_f32_s32:
5528 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5529 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5530 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
5531 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
5532 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: InTy);
5533 if (usgn)
5534 return Builder.CreateUIToFP(V: Ops[0], DestTy: FTy);
5535 return Builder.CreateSIToFP(V: Ops[0], DestTy: FTy);
5536 }
5537 case NEON::BI__builtin_neon_vcvth_f16_u16:
5538 case NEON::BI__builtin_neon_vcvth_f16_u32:
5539 case NEON::BI__builtin_neon_vcvth_f16_u64:
5540 usgn = true;
5541 [[fallthrough]];
5542 case NEON::BI__builtin_neon_vcvth_f16_s16:
5543 case NEON::BI__builtin_neon_vcvth_f16_s32:
5544 case NEON::BI__builtin_neon_vcvth_f16_s64: {
5545 llvm::Type *FTy = HalfTy;
5546 llvm::Type *InTy;
5547 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
5548 InTy = Int64Ty;
5549 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
5550 InTy = Int32Ty;
5551 else
5552 InTy = Int16Ty;
5553 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: InTy);
5554 if (usgn)
5555 return Builder.CreateUIToFP(V: Ops[0], DestTy: FTy);
5556 return Builder.CreateSIToFP(V: Ops[0], DestTy: FTy);
5557 }
5558 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5559 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5560 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5561 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5562 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5563 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5564 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5565 case NEON::BI__builtin_neon_vcvtph_s16_f16: {
5566 llvm::Type *InTy = Int16Ty;
5567 llvm::Type* FTy = HalfTy;
5568 llvm::Type *Tys[2] = {InTy, FTy};
5569 switch (BuiltinID) {
5570 default: llvm_unreachable("missing builtin ID in switch!");
5571 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5572 Int = Intrinsic::aarch64_neon_fcvtau; break;
5573 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5574 Int = Intrinsic::aarch64_neon_fcvtmu; break;
5575 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5576 Int = Intrinsic::aarch64_neon_fcvtnu; break;
5577 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5578 Int = Intrinsic::aarch64_neon_fcvtpu; break;
5579 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5580 Int = Intrinsic::aarch64_neon_fcvtas; break;
5581 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5582 Int = Intrinsic::aarch64_neon_fcvtms; break;
5583 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5584 Int = Intrinsic::aarch64_neon_fcvtns; break;
5585 case NEON::BI__builtin_neon_vcvtph_s16_f16:
5586 Int = Intrinsic::aarch64_neon_fcvtps; break;
5587 }
5588 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fcvt");
5589 }
5590 case NEON::BI__builtin_neon_vcaleh_f16:
5591 case NEON::BI__builtin_neon_vcalth_f16:
5592 case NEON::BI__builtin_neon_vcageh_f16:
5593 case NEON::BI__builtin_neon_vcagth_f16: {
5594 llvm::Type* InTy = Int32Ty;
5595 llvm::Type* FTy = HalfTy;
5596 llvm::Type *Tys[2] = {InTy, FTy};
5597 switch (BuiltinID) {
5598 default: llvm_unreachable("missing builtin ID in switch!");
5599 case NEON::BI__builtin_neon_vcageh_f16:
5600 Int = Intrinsic::aarch64_neon_facge; break;
5601 case NEON::BI__builtin_neon_vcagth_f16:
5602 Int = Intrinsic::aarch64_neon_facgt; break;
5603 case NEON::BI__builtin_neon_vcaleh_f16:
5604 Int = Intrinsic::aarch64_neon_facge; std::swap(a&: Ops[0], b&: Ops[1]); break;
5605 case NEON::BI__builtin_neon_vcalth_f16:
5606 Int = Intrinsic::aarch64_neon_facgt; std::swap(a&: Ops[0], b&: Ops[1]); break;
5607 }
5608 Ops[0] = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "facg");
5609 return Builder.CreateTrunc(V: Ops[0], DestTy: Int16Ty);
5610 }
5611 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5612 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
5613 llvm::Type* InTy = Int32Ty;
5614 llvm::Type* FTy = HalfTy;
5615 llvm::Type *Tys[2] = {InTy, FTy};
5616 switch (BuiltinID) {
5617 default: llvm_unreachable("missing builtin ID in switch!");
5618 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5619 Int = Intrinsic::aarch64_neon_vcvtfp2fxs; break;
5620 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
5621 Int = Intrinsic::aarch64_neon_vcvtfp2fxu; break;
5622 }
5623 Ops[0] = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fcvth_n");
5624 return Builder.CreateTrunc(V: Ops[0], DestTy: Int16Ty);
5625 }
5626 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5627 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
5628 llvm::Type* FTy = HalfTy;
5629 llvm::Type* InTy = Int32Ty;
5630 llvm::Type *Tys[2] = {FTy, InTy};
5631 switch (BuiltinID) {
5632 default: llvm_unreachable("missing builtin ID in switch!");
5633 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5634 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
5635 Ops[0] = Builder.CreateSExt(V: Ops[0], DestTy: InTy, Name: "sext");
5636 break;
5637 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
5638 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
5639 Ops[0] = Builder.CreateZExt(V: Ops[0], DestTy: InTy);
5640 break;
5641 }
5642 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "fcvth_n");
5643 }
5644 case NEON::BI__builtin_neon_vpaddd_s64: {
5645 // TODO: Isn't this handled by
5646 // EmitCommonNeonSISDBuiltinExpr?
5647 auto *Ty = llvm::FixedVectorType::get(ElementType: Int64Ty, NumElts: 2);
5648 // The vector is v2f64, so make sure it's bitcast to that.
5649 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty, Name: "v2i64");
5650 llvm::Value *Idx0 = llvm::ConstantInt::get(Ty: SizeTy, V: 0);
5651 llvm::Value *Idx1 = llvm::ConstantInt::get(Ty: SizeTy, V: 1);
5652 Value *Op0 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx0, Name: "lane0");
5653 Value *Op1 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx1, Name: "lane1");
5654 // Pairwise addition of a v2f64 into a scalar f64.
5655 return Builder.CreateAdd(LHS: Op0, RHS: Op1, Name: "vpaddd");
5656 }
5657 case NEON::BI__builtin_neon_vpaddd_f64: {
5658 auto *Ty = llvm::FixedVectorType::get(ElementType: DoubleTy, NumElts: 2);
5659 // The vector is v2f64, so make sure it's bitcast to that.
5660 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty, Name: "v2f64");
5661 llvm::Value *Idx0 = llvm::ConstantInt::get(Ty: SizeTy, V: 0);
5662 llvm::Value *Idx1 = llvm::ConstantInt::get(Ty: SizeTy, V: 1);
5663 Value *Op0 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx0, Name: "lane0");
5664 Value *Op1 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx1, Name: "lane1");
5665 // Pairwise addition of a v2f64 into a scalar f64.
5666 return Builder.CreateFAdd(L: Op0, R: Op1, Name: "vpaddd");
5667 }
5668 case NEON::BI__builtin_neon_vpadds_f32: {
5669 auto *Ty = llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 2);
5670 // The vector is v2f32, so make sure it's bitcast to that.
5671 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty, Name: "v2f32");
5672 llvm::Value *Idx0 = llvm::ConstantInt::get(Ty: SizeTy, V: 0);
5673 llvm::Value *Idx1 = llvm::ConstantInt::get(Ty: SizeTy, V: 1);
5674 Value *Op0 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx0, Name: "lane0");
5675 Value *Op1 = Builder.CreateExtractElement(Vec: Ops[0], Idx: Idx1, Name: "lane1");
5676 // Pairwise addition of a v2f32 into a scalar f32.
5677 return Builder.CreateFAdd(L: Op0, R: Op1, Name: "vpaddd");
5678 }
5679 case NEON::BI__builtin_neon_vceqzd_s64:
5680 return EmitAArch64CompareBuiltinExpr(
5681 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5682 Pred: ICmpInst::ICMP_EQ, Name: "vceqz");
5683 case NEON::BI__builtin_neon_vceqzd_f64:
5684 case NEON::BI__builtin_neon_vceqzs_f32:
5685 case NEON::BI__builtin_neon_vceqzh_f16:
5686 return EmitAArch64CompareBuiltinExpr(
5687 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5688 Pred: ICmpInst::FCMP_OEQ, Name: "vceqz");
5689 case NEON::BI__builtin_neon_vcgezd_s64:
5690 return EmitAArch64CompareBuiltinExpr(
5691 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5692 Pred: ICmpInst::ICMP_SGE, Name: "vcgez");
5693 case NEON::BI__builtin_neon_vcgezd_f64:
5694 case NEON::BI__builtin_neon_vcgezs_f32:
5695 case NEON::BI__builtin_neon_vcgezh_f16:
5696 return EmitAArch64CompareBuiltinExpr(
5697 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5698 Pred: ICmpInst::FCMP_OGE, Name: "vcgez");
5699 case NEON::BI__builtin_neon_vclezd_s64:
5700 return EmitAArch64CompareBuiltinExpr(
5701 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5702 Pred: ICmpInst::ICMP_SLE, Name: "vclez");
5703 case NEON::BI__builtin_neon_vclezd_f64:
5704 case NEON::BI__builtin_neon_vclezs_f32:
5705 case NEON::BI__builtin_neon_vclezh_f16:
5706 return EmitAArch64CompareBuiltinExpr(
5707 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5708 Pred: ICmpInst::FCMP_OLE, Name: "vclez");
5709 case NEON::BI__builtin_neon_vcgtzd_s64:
5710 return EmitAArch64CompareBuiltinExpr(
5711 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5712 Pred: ICmpInst::ICMP_SGT, Name: "vcgtz");
5713 case NEON::BI__builtin_neon_vcgtzd_f64:
5714 case NEON::BI__builtin_neon_vcgtzs_f32:
5715 case NEON::BI__builtin_neon_vcgtzh_f16:
5716 return EmitAArch64CompareBuiltinExpr(
5717 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5718 Pred: ICmpInst::FCMP_OGT, Name: "vcgtz");
5719 case NEON::BI__builtin_neon_vcltzd_s64:
5720 return EmitAArch64CompareBuiltinExpr(
5721 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5722 Pred: ICmpInst::ICMP_SLT, Name: "vcltz");
5723
5724 case NEON::BI__builtin_neon_vcltzd_f64:
5725 case NEON::BI__builtin_neon_vcltzs_f32:
5726 case NEON::BI__builtin_neon_vcltzh_f16:
5727 return EmitAArch64CompareBuiltinExpr(
5728 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5729 Pred: ICmpInst::FCMP_OLT, Name: "vcltz");
5730
5731 case NEON::BI__builtin_neon_vceqzd_u64: {
5732 return EmitAArch64CompareBuiltinExpr(
5733 Op: Ops[0], Ty: ConvertType(T: E->getCallReturnType(Ctx: getContext())),
5734 Pred: ICmpInst::ICMP_EQ, Name: "vceqzd");
5735 }
5736 case NEON::BI__builtin_neon_vceqd_f64:
5737 case NEON::BI__builtin_neon_vcled_f64:
5738 case NEON::BI__builtin_neon_vcltd_f64:
5739 case NEON::BI__builtin_neon_vcged_f64:
5740 case NEON::BI__builtin_neon_vcgtd_f64: {
5741 llvm::CmpInst::Predicate P;
5742 switch (BuiltinID) {
5743 default: llvm_unreachable("missing builtin ID in switch!");
5744 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
5745 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
5746 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
5747 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
5748 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
5749 }
5750 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
5751 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: DoubleTy);
5752 if (P == llvm::FCmpInst::FCMP_OEQ)
5753 Ops[0] = Builder.CreateFCmp(P, LHS: Ops[0], RHS: Ops[1]);
5754 else
5755 Ops[0] = Builder.CreateFCmpS(P, LHS: Ops[0], RHS: Ops[1]);
5756 return Builder.CreateSExt(V: Ops[0], DestTy: Int64Ty, Name: "vcmpd");
5757 }
5758 case NEON::BI__builtin_neon_vceqs_f32:
5759 case NEON::BI__builtin_neon_vcles_f32:
5760 case NEON::BI__builtin_neon_vclts_f32:
5761 case NEON::BI__builtin_neon_vcges_f32:
5762 case NEON::BI__builtin_neon_vcgts_f32: {
5763 llvm::CmpInst::Predicate P;
5764 switch (BuiltinID) {
5765 default: llvm_unreachable("missing builtin ID in switch!");
5766 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
5767 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
5768 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
5769 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
5770 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
5771 }
5772 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: FloatTy);
5773 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: FloatTy);
5774 if (P == llvm::FCmpInst::FCMP_OEQ)
5775 Ops[0] = Builder.CreateFCmp(P, LHS: Ops[0], RHS: Ops[1]);
5776 else
5777 Ops[0] = Builder.CreateFCmpS(P, LHS: Ops[0], RHS: Ops[1]);
5778 return Builder.CreateSExt(V: Ops[0], DestTy: Int32Ty, Name: "vcmpd");
5779 }
5780 case NEON::BI__builtin_neon_vceqh_f16:
5781 case NEON::BI__builtin_neon_vcleh_f16:
5782 case NEON::BI__builtin_neon_vclth_f16:
5783 case NEON::BI__builtin_neon_vcgeh_f16:
5784 case NEON::BI__builtin_neon_vcgth_f16: {
5785 llvm::CmpInst::Predicate P;
5786 switch (BuiltinID) {
5787 default: llvm_unreachable("missing builtin ID in switch!");
5788 case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ; break;
5789 case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE; break;
5790 case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT; break;
5791 case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE; break;
5792 case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT; break;
5793 }
5794 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: HalfTy);
5795 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: HalfTy);
5796 if (P == llvm::FCmpInst::FCMP_OEQ)
5797 Ops[0] = Builder.CreateFCmp(P, LHS: Ops[0], RHS: Ops[1]);
5798 else
5799 Ops[0] = Builder.CreateFCmpS(P, LHS: Ops[0], RHS: Ops[1]);
5800 return Builder.CreateSExt(V: Ops[0], DestTy: Int16Ty, Name: "vcmpd");
5801 }
5802 case NEON::BI__builtin_neon_vceqd_s64:
5803 case NEON::BI__builtin_neon_vceqd_u64:
5804 case NEON::BI__builtin_neon_vcgtd_s64:
5805 case NEON::BI__builtin_neon_vcgtd_u64:
5806 case NEON::BI__builtin_neon_vcltd_s64:
5807 case NEON::BI__builtin_neon_vcltd_u64:
5808 case NEON::BI__builtin_neon_vcged_u64:
5809 case NEON::BI__builtin_neon_vcged_s64:
5810 case NEON::BI__builtin_neon_vcled_u64:
5811 case NEON::BI__builtin_neon_vcled_s64: {
5812 llvm::CmpInst::Predicate P;
5813 switch (BuiltinID) {
5814 default: llvm_unreachable("missing builtin ID in switch!");
5815 case NEON::BI__builtin_neon_vceqd_s64:
5816 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
5817 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
5818 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
5819 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
5820 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
5821 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
5822 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
5823 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
5824 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
5825 }
5826 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Int64Ty);
5827 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty);
5828 Ops[0] = Builder.CreateICmp(P, LHS: Ops[0], RHS: Ops[1]);
5829 return Builder.CreateSExt(V: Ops[0], DestTy: Int64Ty, Name: "vceqd");
5830 }
5831 case NEON::BI__builtin_neon_vnegd_s64:
5832 return Builder.CreateNeg(V: Ops[0], Name: "vnegd");
5833 case NEON::BI__builtin_neon_vnegh_f16:
5834 return Builder.CreateFNeg(V: Ops[0], Name: "vnegh");
5835 case NEON::BI__builtin_neon_vtstd_s64:
5836 case NEON::BI__builtin_neon_vtstd_u64: {
5837 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Int64Ty);
5838 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty);
5839 Ops[0] = Builder.CreateAnd(LHS: Ops[0], RHS: Ops[1]);
5840 Ops[0] = Builder.CreateICmp(P: ICmpInst::ICMP_NE, LHS: Ops[0],
5841 RHS: llvm::Constant::getNullValue(Ty: Int64Ty));
5842 return Builder.CreateSExt(V: Ops[0], DestTy: Int64Ty, Name: "vtstd");
5843 }
5844 case NEON::BI__builtin_neon_vset_lane_i8:
5845 case NEON::BI__builtin_neon_vset_lane_i16:
5846 case NEON::BI__builtin_neon_vset_lane_i32:
5847 case NEON::BI__builtin_neon_vset_lane_i64:
5848 case NEON::BI__builtin_neon_vset_lane_bf16:
5849 case NEON::BI__builtin_neon_vset_lane_f32:
5850 case NEON::BI__builtin_neon_vsetq_lane_i8:
5851 case NEON::BI__builtin_neon_vsetq_lane_i16:
5852 case NEON::BI__builtin_neon_vsetq_lane_i32:
5853 case NEON::BI__builtin_neon_vsetq_lane_i64:
5854 case NEON::BI__builtin_neon_vsetq_lane_bf16:
5855 case NEON::BI__builtin_neon_vsetq_lane_f32:
5856 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5857 case NEON::BI__builtin_neon_vset_lane_f64:
5858 // The vector type needs a cast for the v1f64 variant.
5859 Ops[1] =
5860 Builder.CreateBitCast(V: Ops[1], DestTy: llvm::FixedVectorType::get(ElementType: DoubleTy, NumElts: 1));
5861 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5862 case NEON::BI__builtin_neon_vset_lane_mf8:
5863 case NEON::BI__builtin_neon_vsetq_lane_mf8:
5864 // The input vector type needs a cast to scalar type.
5865 Ops[0] =
5866 Builder.CreateBitCast(V: Ops[0], DestTy: llvm::Type::getInt8Ty(C&: getLLVMContext()));
5867 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5868 case NEON::BI__builtin_neon_vsetq_lane_f64:
5869 // The vector type needs a cast for the v2f64 variant.
5870 Ops[1] =
5871 Builder.CreateBitCast(V: Ops[1], DestTy: llvm::FixedVectorType::get(ElementType: DoubleTy, NumElts: 2));
5872 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vset_lane");
5873
5874 case NEON::BI__builtin_neon_vget_lane_i8:
5875 case NEON::BI__builtin_neon_vdupb_lane_i8:
5876 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5877 case NEON::BI__builtin_neon_vgetq_lane_i8:
5878 case NEON::BI__builtin_neon_vdupb_laneq_i8:
5879 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5880 case NEON::BI__builtin_neon_vget_lane_mf8:
5881 case NEON::BI__builtin_neon_vdupb_lane_mf8:
5882 case NEON::BI__builtin_neon_vgetq_lane_mf8:
5883 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
5884 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5885 case NEON::BI__builtin_neon_vget_lane_i16:
5886 case NEON::BI__builtin_neon_vduph_lane_i16:
5887 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5888 case NEON::BI__builtin_neon_vgetq_lane_i16:
5889 case NEON::BI__builtin_neon_vduph_laneq_i16:
5890 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5891 case NEON::BI__builtin_neon_vget_lane_i32:
5892 case NEON::BI__builtin_neon_vdups_lane_i32:
5893 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5894 case NEON::BI__builtin_neon_vdups_lane_f32:
5895 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vdups_lane");
5896 case NEON::BI__builtin_neon_vgetq_lane_i32:
5897 case NEON::BI__builtin_neon_vdups_laneq_i32:
5898 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5899 case NEON::BI__builtin_neon_vget_lane_i64:
5900 case NEON::BI__builtin_neon_vdupd_lane_i64:
5901 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5902 case NEON::BI__builtin_neon_vdupd_lane_f64:
5903 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vdupd_lane");
5904 case NEON::BI__builtin_neon_vgetq_lane_i64:
5905 case NEON::BI__builtin_neon_vdupd_laneq_i64:
5906 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5907 case NEON::BI__builtin_neon_vget_lane_f32:
5908 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5909 case NEON::BI__builtin_neon_vget_lane_f64:
5910 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
5911 case NEON::BI__builtin_neon_vgetq_lane_f32:
5912 case NEON::BI__builtin_neon_vdups_laneq_f32:
5913 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5914 case NEON::BI__builtin_neon_vgetq_lane_f64:
5915 case NEON::BI__builtin_neon_vdupd_laneq_f64:
5916 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
5917 case NEON::BI__builtin_neon_vaddh_f16:
5918 return Builder.CreateFAdd(L: Ops[0], R: Ops[1], Name: "vaddh");
5919 case NEON::BI__builtin_neon_vsubh_f16:
5920 return Builder.CreateFSub(L: Ops[0], R: Ops[1], Name: "vsubh");
5921 case NEON::BI__builtin_neon_vmulh_f16:
5922 return Builder.CreateFMul(L: Ops[0], R: Ops[1], Name: "vmulh");
5923 case NEON::BI__builtin_neon_vdivh_f16:
5924 return Builder.CreateFDiv(L: Ops[0], R: Ops[1], Name: "vdivh");
5925 case NEON::BI__builtin_neon_vfmah_f16:
5926 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5927 return emitCallMaybeConstrainedFPBuiltin(
5928 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty: HalfTy,
5929 Args: {Ops[1], Ops[2], Ops[0]});
5930 case NEON::BI__builtin_neon_vfmsh_f16: {
5931 Value *Neg = Builder.CreateFNeg(V: Ops[1], Name: "vsubh");
5932
5933 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5934 return emitCallMaybeConstrainedFPBuiltin(
5935 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty: HalfTy,
5936 Args: {Neg, Ops[2], Ops[0]});
5937 }
5938 case NEON::BI__builtin_neon_vaddd_s64:
5939 case NEON::BI__builtin_neon_vaddd_u64:
5940 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1], Name: "vaddd");
5941 case NEON::BI__builtin_neon_vsubd_s64:
5942 case NEON::BI__builtin_neon_vsubd_u64:
5943 return Builder.CreateSub(LHS: Ops[0], RHS: Ops[1], Name: "vsubd");
5944 case NEON::BI__builtin_neon_vqdmlalh_s16:
5945 case NEON::BI__builtin_neon_vqdmlslh_s16: {
5946 SmallVector<Value *, 2> ProductOps;
5947 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[1]));
5948 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[2]));
5949 auto *VTy = llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: 4);
5950 Ops[1] = EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmull, Tys: VTy),
5951 Ops&: ProductOps, name: "vqdmlXl");
5952 Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
5953 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: CI, Name: "lane0");
5954
5955 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
5956 ? Intrinsic::aarch64_neon_sqadd
5957 : Intrinsic::aarch64_neon_sqsub;
5958 // Drop the 2nd multiplication argument before the accumulation
5959 Ops.pop_back();
5960 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccumInt, Tys: Int32Ty), Ops, name: "vqdmlXl");
5961 }
5962 case NEON::BI__builtin_neon_vqshlud_n_s64: {
5963 Ops[1] = Builder.CreateZExt(V: Ops[1], DestTy: Int64Ty);
5964 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqshlu, Tys: Int64Ty),
5965 Ops, name: "vqshlu_n");
5966 }
5967 case NEON::BI__builtin_neon_vqshld_n_u64:
5968 case NEON::BI__builtin_neon_vqshld_n_s64: {
5969 Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
5970 ? Intrinsic::aarch64_neon_uqshl
5971 : Intrinsic::aarch64_neon_sqshl;
5972 Ops[1] = Builder.CreateZExt(V: Ops[1], DestTy: Int64Ty);
5973 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Int64Ty), Ops, name: "vqshl_n");
5974 }
5975 case NEON::BI__builtin_neon_vrshrd_n_u64:
5976 case NEON::BI__builtin_neon_vrshrd_n_s64: {
5977 Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
5978 ? Intrinsic::aarch64_neon_urshl
5979 : Intrinsic::aarch64_neon_srshl;
5980 int SV = cast<ConstantInt>(Val: Ops[1])->getSExtValue();
5981 Ops[1] = ConstantInt::get(Ty: Int64Ty, V: -SV);
5982 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Int64Ty), Ops, name: "vrshr_n");
5983 }
5984 case NEON::BI__builtin_neon_vrsrad_n_u64:
5985 case NEON::BI__builtin_neon_vrsrad_n_s64: {
5986 Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
5987 ? Intrinsic::aarch64_neon_urshl
5988 : Intrinsic::aarch64_neon_srshl;
5989 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty);
5990 Ops[2] = Builder.CreateNeg(V: Ops[2]);
5991 Ops[1] = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Int, Tys: Int64Ty),
5992 Args: {Ops[1], Builder.CreateSExt(V: Ops[2], DestTy: Int64Ty)});
5993 return Builder.CreateAdd(LHS: Ops[0], RHS: Builder.CreateBitCast(V: Ops[1], DestTy: Int64Ty));
5994 }
5995 case NEON::BI__builtin_neon_vshld_n_s64:
5996 case NEON::BI__builtin_neon_vshld_n_u64: {
5997 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[1]);
5998 return Builder.CreateShl(
5999 LHS: Ops[0], RHS: ConstantInt::get(Ty: Int64Ty, V: Amt->getZExtValue()), Name: "shld_n");
6000 }
6001 case NEON::BI__builtin_neon_vshrd_n_s64: {
6002 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[1]);
6003 return Builder.CreateAShr(
6004 LHS: Ops[0], RHS: ConstantInt::get(Ty: Int64Ty, V: std::min(a: static_cast<uint64_t>(63),
6005 b: Amt->getZExtValue())),
6006 Name: "shrd_n");
6007 }
6008 case NEON::BI__builtin_neon_vshrd_n_u64: {
6009 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[1]);
6010 uint64_t ShiftAmt = Amt->getZExtValue();
6011 // Right-shifting an unsigned value by its size yields 0.
6012 if (ShiftAmt == 64)
6013 return ConstantInt::get(Ty: Int64Ty, V: 0);
6014 return Builder.CreateLShr(LHS: Ops[0], RHS: ConstantInt::get(Ty: Int64Ty, V: ShiftAmt),
6015 Name: "shrd_n");
6016 }
6017 case NEON::BI__builtin_neon_vsrad_n_s64: {
6018 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[2]);
6019 Ops[1] = Builder.CreateAShr(
6020 LHS: Ops[1], RHS: ConstantInt::get(Ty: Int64Ty, V: std::min(a: static_cast<uint64_t>(63),
6021 b: Amt->getZExtValue())),
6022 Name: "shrd_n");
6023 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
6024 }
6025 case NEON::BI__builtin_neon_vsrad_n_u64: {
6026 llvm::ConstantInt *Amt = cast<ConstantInt>(Val: Ops[2]);
6027 uint64_t ShiftAmt = Amt->getZExtValue();
6028 // Right-shifting an unsigned value by its size yields 0.
6029 // As Op + 0 = Op, return Ops[0] directly.
6030 if (ShiftAmt == 64)
6031 return Ops[0];
6032 Ops[1] = Builder.CreateLShr(LHS: Ops[1], RHS: ConstantInt::get(Ty: Int64Ty, V: ShiftAmt),
6033 Name: "shrd_n");
6034 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
6035 }
6036 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
6037 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
6038 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
6039 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
6040 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "lane");
6041 SmallVector<Value *, 2> ProductOps;
6042 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[1]));
6043 ProductOps.push_back(Elt: vectorWrapScalar16(Op: Ops[2]));
6044 auto *VTy = llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: 4);
6045 Ops[1] = EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmull, Tys: VTy),
6046 Ops&: ProductOps, name: "vqdmlXl");
6047 Constant *CI = ConstantInt::get(Ty: SizeTy, V: 0);
6048 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: CI, Name: "lane0");
6049 // Drop lane-selection and the corresponding vector argument (these have
6050 // already been used)
6051 Ops.pop_back_n(NumItems: 2);
6052
6053 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
6054 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
6055 ? Intrinsic::aarch64_neon_sqadd
6056 : Intrinsic::aarch64_neon_sqsub;
6057 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccInt, Tys: Int32Ty), Ops, name: "vqdmlXl");
6058 }
6059 case NEON::BI__builtin_neon_vqdmlals_s32:
6060 case NEON::BI__builtin_neon_vqdmlsls_s32: {
6061 SmallVector<Value *, 2> ProductOps;
6062 ProductOps.push_back(Elt: Ops[1]);
6063 ProductOps.push_back(Elt: Ops[2]);
6064 Ops[1] =
6065 EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmulls_scalar),
6066 Ops&: ProductOps, name: "vqdmlXl");
6067
6068 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
6069 ? Intrinsic::aarch64_neon_sqadd
6070 : Intrinsic::aarch64_neon_sqsub;
6071 // Drop the 2nd multiplication argument before the accumulation
6072 Ops.pop_back();
6073 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccumInt, Tys: Int64Ty), Ops, name: "vqdmlXl");
6074 }
6075 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
6076 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
6077 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
6078 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
6079 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "lane");
6080 SmallVector<Value *, 2> ProductOps;
6081 ProductOps.push_back(Elt: Ops[1]);
6082 ProductOps.push_back(Elt: Ops[2]);
6083 Ops[1] =
6084 EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_sqdmulls_scalar),
6085 Ops&: ProductOps, name: "vqdmlXl");
6086 // Drop lane-selection and the corresponding vector argument (these have
6087 // already been used)
6088 Ops.pop_back_n(NumItems: 2);
6089
6090 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
6091 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
6092 ? Intrinsic::aarch64_neon_sqadd
6093 : Intrinsic::aarch64_neon_sqsub;
6094 return EmitNeonCall(F: CGM.getIntrinsic(IID: AccInt, Tys: Int64Ty), Ops, name: "vqdmlXl");
6095 }
6096 case NEON::BI__builtin_neon_vget_lane_bf16:
6097 case NEON::BI__builtin_neon_vduph_lane_bf16:
6098 case NEON::BI__builtin_neon_vduph_lane_f16: {
6099 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vget_lane");
6100 }
6101 case NEON::BI__builtin_neon_vgetq_lane_bf16:
6102 case NEON::BI__builtin_neon_vduph_laneq_bf16:
6103 case NEON::BI__builtin_neon_vduph_laneq_f16: {
6104 return Builder.CreateExtractElement(Vec: Ops[0], Idx: Ops[1], Name: "vgetq_lane");
6105 }
6106 case NEON::BI__builtin_neon_vcvt_bf16_f32: {
6107 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6108 llvm::Type *V4BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 4);
6109 return Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F32), DestTy: V4BF16);
6110 }
6111 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32: {
6112 SmallVector<int, 16> ConcatMask(8);
6113 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
6114 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6115 llvm::Type *V4BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 4);
6116 llvm::Value *Trunc =
6117 Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F32), DestTy: V4BF16);
6118 return Builder.CreateShuffleVector(
6119 V1: Trunc, V2: ConstantAggregateZero::get(Ty: V4BF16), Mask: ConcatMask);
6120 }
6121 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32: {
6122 SmallVector<int, 16> ConcatMask(8);
6123 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
6124 SmallVector<int, 16> LoMask(4);
6125 std::iota(first: LoMask.begin(), last: LoMask.end(), value: 0);
6126 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6127 llvm::Type *V4BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 4);
6128 llvm::Type *V8BF16 = FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 8);
6129 llvm::Value *Inactive = Builder.CreateShuffleVector(
6130 V: Builder.CreateBitCast(V: Ops[0], DestTy: V8BF16), Mask: LoMask);
6131 llvm::Value *Trunc =
6132 Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[1], DestTy: V4F32), DestTy: V4BF16);
6133 return Builder.CreateShuffleVector(V1: Inactive, V2: Trunc, Mask: ConcatMask);
6134 }
6135 case NEON::BI__builtin_neon_vcvt_f16_f32: {
6136 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6137 llvm::Type *V4F16 = FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 4);
6138 return Builder.CreateFPTrunc(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F32), DestTy: V4F16);
6139 }
6140 case NEON::BI__builtin_neon_vcvt_f32_f16: {
6141 llvm::Type *V4F32 = FixedVectorType::get(ElementType: Builder.getFloatTy(), NumElts: 4);
6142 llvm::Type *V4F16 = FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 4);
6143 return Builder.CreateFPExt(V: Builder.CreateBitCast(V: Ops[0], DestTy: V4F16), DestTy: V4F32);
6144 }
6145
6146 case clang::AArch64::BI_InterlockedAdd:
6147 case clang::AArch64::BI_InterlockedAdd_acq:
6148 case clang::AArch64::BI_InterlockedAdd_rel:
6149 case clang::AArch64::BI_InterlockedAdd_nf:
6150 case clang::AArch64::BI_InterlockedAdd64:
6151 case clang::AArch64::BI_InterlockedAdd64_acq:
6152 case clang::AArch64::BI_InterlockedAdd64_rel:
6153 case clang::AArch64::BI_InterlockedAdd64_nf: {
6154 Address DestAddr = CheckAtomicAlignment(CGF&: *this, E);
6155 Value *Val = Ops[1];
6156 llvm::AtomicOrdering Ordering;
6157 switch (BuiltinID) {
6158 case clang::AArch64::BI_InterlockedAdd:
6159 case clang::AArch64::BI_InterlockedAdd64:
6160 Ordering = llvm::AtomicOrdering::SequentiallyConsistent;
6161 break;
6162 case clang::AArch64::BI_InterlockedAdd_acq:
6163 case clang::AArch64::BI_InterlockedAdd64_acq:
6164 Ordering = llvm::AtomicOrdering::Acquire;
6165 break;
6166 case clang::AArch64::BI_InterlockedAdd_rel:
6167 case clang::AArch64::BI_InterlockedAdd64_rel:
6168 Ordering = llvm::AtomicOrdering::Release;
6169 break;
6170 case clang::AArch64::BI_InterlockedAdd_nf:
6171 case clang::AArch64::BI_InterlockedAdd64_nf:
6172 Ordering = llvm::AtomicOrdering::Monotonic;
6173 break;
6174 default:
6175 llvm_unreachable("missing builtin ID in switch!");
6176 }
6177 AtomicRMWInst *RMWI =
6178 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Addr: DestAddr, Val, Ordering);
6179 return Builder.CreateAdd(LHS: RMWI, RHS: Val);
6180 }
6181 }
6182
6183 llvm::FixedVectorType *VTy = GetNeonType(CGF: this, TypeFlags: Type);
6184 llvm::Type *Ty = VTy;
6185 if (!Ty)
6186 return nullptr;
6187
6188 bool ExtractLow = false;
6189 bool ExtendLaneArg = false;
6190 switch (BuiltinID) {
6191 default: return nullptr;
6192 case NEON::BI__builtin_neon_vbsl_v:
6193 case NEON::BI__builtin_neon_vbslq_v: {
6194 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
6195 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: BitTy, Name: "vbsl");
6196 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: BitTy, Name: "vbsl");
6197 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: BitTy, Name: "vbsl");
6198
6199 Ops[1] = Builder.CreateAnd(LHS: Ops[0], RHS: Ops[1], Name: "vbsl");
6200 Ops[2] = Builder.CreateAnd(LHS: Builder.CreateNot(V: Ops[0]), RHS: Ops[2], Name: "vbsl");
6201 Ops[0] = Builder.CreateOr(LHS: Ops[1], RHS: Ops[2], Name: "vbsl");
6202 return Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6203 }
6204 case NEON::BI__builtin_neon_vfma_lane_v:
6205 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
6206 // The ARM builtins (and instructions) have the addend as the first
6207 // operand, but the 'fma' intrinsics have it last. Swap it around here.
6208 Value *Addend = Ops[0];
6209 Value *Multiplicand = Ops[1];
6210 Value *LaneSource = Ops[2];
6211 Ops[0] = Multiplicand;
6212 Ops[1] = LaneSource;
6213 Ops[2] = Addend;
6214
6215 // Now adjust things to handle the lane access.
6216 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
6217 ? llvm::FixedVectorType::get(ElementType: VTy->getElementType(),
6218 NumElts: VTy->getNumElements() / 2)
6219 : VTy;
6220 llvm::Constant *cst = cast<Constant>(Val: Ops[3]);
6221 Value *SV = llvm::ConstantVector::getSplat(EC: VTy->getElementCount(), Elt: cst);
6222 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: SourceTy);
6223 Ops[1] = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[1], Mask: SV, Name: "lane");
6224
6225 Ops.pop_back();
6226 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
6227 : Intrinsic::fma;
6228 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "fmla");
6229 }
6230 case NEON::BI__builtin_neon_vfma_laneq_v: {
6231 auto *VTy = cast<llvm::FixedVectorType>(Val: Ty);
6232 // v1f64 fma should be mapped to Neon scalar f64 fma
6233 if (VTy && VTy->getElementType() == DoubleTy) {
6234 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
6235 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: DoubleTy);
6236 llvm::FixedVectorType *VTy =
6237 GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float64, false, true));
6238 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: VTy);
6239 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "extract");
6240 Value *Result;
6241 Result = emitCallMaybeConstrainedFPBuiltin(
6242 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma,
6243 Ty: DoubleTy, Args: {Ops[1], Ops[2], Ops[0]});
6244 return Builder.CreateBitCast(V: Result, DestTy: Ty);
6245 }
6246 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6247 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6248
6249 auto *STy = llvm::FixedVectorType::get(ElementType: VTy->getElementType(),
6250 NumElts: VTy->getNumElements() * 2);
6251 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: STy);
6252 Value *SV = llvm::ConstantVector::getSplat(EC: VTy->getElementCount(),
6253 Elt: cast<ConstantInt>(Val: Ops[3]));
6254 Ops[2] = Builder.CreateShuffleVector(V1: Ops[2], V2: Ops[2], Mask: SV, Name: "lane");
6255
6256 return emitCallMaybeConstrainedFPBuiltin(
6257 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
6258 Args: {Ops[2], Ops[1], Ops[0]});
6259 }
6260 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
6261 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6262 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6263
6264 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6265 Ops[2] = EmitNeonSplat(V: Ops[2], C: cast<ConstantInt>(Val: Ops[3]));
6266 return emitCallMaybeConstrainedFPBuiltin(
6267 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
6268 Args: {Ops[2], Ops[1], Ops[0]});
6269 }
6270 case NEON::BI__builtin_neon_vfmah_lane_f16:
6271 case NEON::BI__builtin_neon_vfmas_lane_f32:
6272 case NEON::BI__builtin_neon_vfmah_laneq_f16:
6273 case NEON::BI__builtin_neon_vfmas_laneq_f32:
6274 case NEON::BI__builtin_neon_vfmad_lane_f64:
6275 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
6276 llvm::Type *Ty = ConvertType(T: E->getCallReturnType(Ctx: getContext()));
6277 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: Ops[3], Name: "extract");
6278 return emitCallMaybeConstrainedFPBuiltin(
6279 CGF&: *this, IntrinsicID: Intrinsic::fma, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma, Ty,
6280 Args: {Ops[1], Ops[2], Ops[0]});
6281 }
6282 case NEON::BI__builtin_neon_vmull_v:
6283 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6284 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
6285 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
6286 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmull");
6287 case NEON::BI__builtin_neon_vmax_v:
6288 case NEON::BI__builtin_neon_vmaxq_v:
6289 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6290 Int = usgn ? Intrinsic::umax : Intrinsic::smax;
6291 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
6292 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmax");
6293 case NEON::BI__builtin_neon_vmaxh_f16: {
6294 Int = Intrinsic::aarch64_neon_fmax;
6295 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmax");
6296 }
6297 case NEON::BI__builtin_neon_vmin_v:
6298 case NEON::BI__builtin_neon_vminq_v:
6299 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6300 Int = usgn ? Intrinsic::umin : Intrinsic::smin;
6301 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
6302 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmin");
6303 case NEON::BI__builtin_neon_vminh_f16: {
6304 Int = Intrinsic::aarch64_neon_fmin;
6305 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmin");
6306 }
6307 case NEON::BI__builtin_neon_vabd_v:
6308 case NEON::BI__builtin_neon_vabdq_v:
6309 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6310 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
6311 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
6312 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vabd");
6313 case NEON::BI__builtin_neon_vpadal_v:
6314 case NEON::BI__builtin_neon_vpadalq_v: {
6315 unsigned ArgElts = VTy->getNumElements();
6316 llvm::IntegerType *EltTy = cast<IntegerType>(Val: VTy->getElementType());
6317 unsigned BitWidth = EltTy->getBitWidth();
6318 auto *ArgTy = llvm::FixedVectorType::get(
6319 ElementType: llvm::IntegerType::get(C&: getLLVMContext(), NumBits: BitWidth / 2), NumElts: 2 * ArgElts);
6320 llvm::Type* Tys[2] = { VTy, ArgTy };
6321 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
6322 SmallVector<llvm::Value*, 1> TmpOps;
6323 TmpOps.push_back(Elt: Ops[1]);
6324 Function *F = CGM.getIntrinsic(IID: Int, Tys);
6325 llvm::Value *tmp = EmitNeonCall(F, Ops&: TmpOps, name: "vpadal");
6326 llvm::Value *addend = Builder.CreateBitCast(V: Ops[0], DestTy: tmp->getType());
6327 return Builder.CreateAdd(LHS: tmp, RHS: addend);
6328 }
6329 case NEON::BI__builtin_neon_vpmin_v:
6330 case NEON::BI__builtin_neon_vpminq_v:
6331 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6332 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
6333 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
6334 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpmin");
6335 case NEON::BI__builtin_neon_vpmax_v:
6336 case NEON::BI__builtin_neon_vpmaxq_v:
6337 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6338 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
6339 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
6340 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpmax");
6341 case NEON::BI__builtin_neon_vminnm_v:
6342 case NEON::BI__builtin_neon_vminnmq_v:
6343 Int = Intrinsic::aarch64_neon_fminnm;
6344 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vminnm");
6345 case NEON::BI__builtin_neon_vminnmh_f16:
6346 Int = Intrinsic::aarch64_neon_fminnm;
6347 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vminnm");
6348 case NEON::BI__builtin_neon_vmaxnm_v:
6349 case NEON::BI__builtin_neon_vmaxnmq_v:
6350 Int = Intrinsic::aarch64_neon_fmaxnm;
6351 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmaxnm");
6352 case NEON::BI__builtin_neon_vmaxnmh_f16:
6353 Int = Intrinsic::aarch64_neon_fmaxnm;
6354 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmaxnm");
6355 case NEON::BI__builtin_neon_vrecpss_f32: {
6356 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_frecps, Tys: FloatTy),
6357 Ops, name: "vrecps");
6358 }
6359 case NEON::BI__builtin_neon_vrecpsd_f64:
6360 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_frecps, Tys: DoubleTy),
6361 Ops, name: "vrecps");
6362 case NEON::BI__builtin_neon_vrecpsh_f16:
6363 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_frecps, Tys: HalfTy),
6364 Ops, name: "vrecps");
6365 case NEON::BI__builtin_neon_vqshrun_n_v:
6366 Int = Intrinsic::aarch64_neon_sqshrun;
6367 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshrun_n");
6368 case NEON::BI__builtin_neon_vqrshrun_n_v:
6369 Int = Intrinsic::aarch64_neon_sqrshrun;
6370 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqrshrun_n");
6371 case NEON::BI__builtin_neon_vqshrn_n_v:
6372 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
6373 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqshrn_n");
6374 case NEON::BI__builtin_neon_vrshrn_n_v:
6375 Int = Intrinsic::aarch64_neon_rshrn;
6376 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrshrn_n");
6377 case NEON::BI__builtin_neon_vqrshrn_n_v:
6378 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
6379 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vqrshrn_n");
6380 case NEON::BI__builtin_neon_vrndah_f16: {
6381 Int = Builder.getIsFPConstrained()
6382 ? Intrinsic::experimental_constrained_round
6383 : Intrinsic::round;
6384 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrnda");
6385 }
6386 case NEON::BI__builtin_neon_vrnda_v:
6387 case NEON::BI__builtin_neon_vrndaq_v: {
6388 Int = Builder.getIsFPConstrained()
6389 ? Intrinsic::experimental_constrained_round
6390 : Intrinsic::round;
6391 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrnda");
6392 }
6393 case NEON::BI__builtin_neon_vrndih_f16: {
6394 Int = Builder.getIsFPConstrained()
6395 ? Intrinsic::experimental_constrained_nearbyint
6396 : Intrinsic::nearbyint;
6397 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndi");
6398 }
6399 case NEON::BI__builtin_neon_vrndmh_f16: {
6400 Int = Builder.getIsFPConstrained()
6401 ? Intrinsic::experimental_constrained_floor
6402 : Intrinsic::floor;
6403 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndm");
6404 }
6405 case NEON::BI__builtin_neon_vrndm_v:
6406 case NEON::BI__builtin_neon_vrndmq_v: {
6407 Int = Builder.getIsFPConstrained()
6408 ? Intrinsic::experimental_constrained_floor
6409 : Intrinsic::floor;
6410 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndm");
6411 }
6412 case NEON::BI__builtin_neon_vrndnh_f16: {
6413 Int = Builder.getIsFPConstrained()
6414 ? Intrinsic::experimental_constrained_roundeven
6415 : Intrinsic::roundeven;
6416 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndn");
6417 }
6418 case NEON::BI__builtin_neon_vrndn_v:
6419 case NEON::BI__builtin_neon_vrndnq_v: {
6420 Int = Builder.getIsFPConstrained()
6421 ? Intrinsic::experimental_constrained_roundeven
6422 : Intrinsic::roundeven;
6423 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndn");
6424 }
6425 case NEON::BI__builtin_neon_vrndns_f32: {
6426 Int = Builder.getIsFPConstrained()
6427 ? Intrinsic::experimental_constrained_roundeven
6428 : Intrinsic::roundeven;
6429 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: FloatTy), Ops, name: "vrndn");
6430 }
6431 case NEON::BI__builtin_neon_vrndph_f16: {
6432 Int = Builder.getIsFPConstrained()
6433 ? Intrinsic::experimental_constrained_ceil
6434 : Intrinsic::ceil;
6435 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndp");
6436 }
6437 case NEON::BI__builtin_neon_vrndp_v:
6438 case NEON::BI__builtin_neon_vrndpq_v: {
6439 Int = Builder.getIsFPConstrained()
6440 ? Intrinsic::experimental_constrained_ceil
6441 : Intrinsic::ceil;
6442 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndp");
6443 }
6444 case NEON::BI__builtin_neon_vrndxh_f16: {
6445 Int = Builder.getIsFPConstrained()
6446 ? Intrinsic::experimental_constrained_rint
6447 : Intrinsic::rint;
6448 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndx");
6449 }
6450 case NEON::BI__builtin_neon_vrndx_v:
6451 case NEON::BI__builtin_neon_vrndxq_v: {
6452 Int = Builder.getIsFPConstrained()
6453 ? Intrinsic::experimental_constrained_rint
6454 : Intrinsic::rint;
6455 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndx");
6456 }
6457 case NEON::BI__builtin_neon_vrndh_f16: {
6458 Int = Builder.getIsFPConstrained()
6459 ? Intrinsic::experimental_constrained_trunc
6460 : Intrinsic::trunc;
6461 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vrndz");
6462 }
6463 case NEON::BI__builtin_neon_vrnd_v:
6464 case NEON::BI__builtin_neon_vrndq_v: {
6465 Int = Builder.getIsFPConstrained()
6466 ? Intrinsic::experimental_constrained_trunc
6467 : Intrinsic::trunc;
6468 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrndz");
6469 }
6470 case NEON::BI__builtin_neon_vcvt_f64_v:
6471 case NEON::BI__builtin_neon_vcvtq_f64_v:
6472 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6473 Ty = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
6474 return usgn ? Builder.CreateUIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt")
6475 : Builder.CreateSIToFP(V: Ops[0], DestTy: Ty, Name: "vcvt");
6476 case NEON::BI__builtin_neon_vcvt_f64_f32: {
6477 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&
6478 "unexpected vcvt_f64_f32 builtin");
6479 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
6480 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: GetNeonType(CGF: this, TypeFlags: SrcFlag));
6481
6482 return Builder.CreateFPExt(V: Ops[0], DestTy: Ty, Name: "vcvt");
6483 }
6484 case NEON::BI__builtin_neon_vcvt_f32_f64: {
6485 assert(Type.getEltType() == NeonTypeFlags::Float32 &&
6486 "unexpected vcvt_f32_f64 builtin");
6487 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
6488 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: GetNeonType(CGF: this, TypeFlags: SrcFlag));
6489
6490 return Builder.CreateFPTrunc(V: Ops[0], DestTy: Ty, Name: "vcvt");
6491 }
6492 case NEON::BI__builtin_neon_vcvta_s16_f16:
6493 case NEON::BI__builtin_neon_vcvta_u16_f16:
6494 case NEON::BI__builtin_neon_vcvta_s32_v:
6495 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
6496 case NEON::BI__builtin_neon_vcvtaq_s32_v:
6497 case NEON::BI__builtin_neon_vcvta_u32_v:
6498 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
6499 case NEON::BI__builtin_neon_vcvtaq_u32_v:
6500 case NEON::BI__builtin_neon_vcvta_s64_v:
6501 case NEON::BI__builtin_neon_vcvtaq_s64_v:
6502 case NEON::BI__builtin_neon_vcvta_u64_v:
6503 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
6504 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
6505 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6506 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvta");
6507 }
6508 case NEON::BI__builtin_neon_vcvtm_s16_f16:
6509 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
6510 case NEON::BI__builtin_neon_vcvtm_u16_f16:
6511 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
6512 case NEON::BI__builtin_neon_vcvtm_s32_v:
6513 case NEON::BI__builtin_neon_vcvtmq_s32_v:
6514 case NEON::BI__builtin_neon_vcvtm_u32_v:
6515 case NEON::BI__builtin_neon_vcvtmq_u32_v:
6516 case NEON::BI__builtin_neon_vcvtm_s64_v:
6517 case NEON::BI__builtin_neon_vcvtmq_s64_v:
6518 case NEON::BI__builtin_neon_vcvtm_u64_v:
6519 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
6520 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
6521 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6522 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtm");
6523 }
6524 case NEON::BI__builtin_neon_vcvtn_s16_f16:
6525 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
6526 case NEON::BI__builtin_neon_vcvtn_u16_f16:
6527 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
6528 case NEON::BI__builtin_neon_vcvtn_s32_v:
6529 case NEON::BI__builtin_neon_vcvtnq_s32_v:
6530 case NEON::BI__builtin_neon_vcvtn_u32_v:
6531 case NEON::BI__builtin_neon_vcvtnq_u32_v:
6532 case NEON::BI__builtin_neon_vcvtn_s64_v:
6533 case NEON::BI__builtin_neon_vcvtnq_s64_v:
6534 case NEON::BI__builtin_neon_vcvtn_u64_v:
6535 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
6536 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
6537 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6538 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtn");
6539 }
6540 case NEON::BI__builtin_neon_vcvtp_s16_f16:
6541 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
6542 case NEON::BI__builtin_neon_vcvtp_u16_f16:
6543 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
6544 case NEON::BI__builtin_neon_vcvtp_s32_v:
6545 case NEON::BI__builtin_neon_vcvtpq_s32_v:
6546 case NEON::BI__builtin_neon_vcvtp_u32_v:
6547 case NEON::BI__builtin_neon_vcvtpq_u32_v:
6548 case NEON::BI__builtin_neon_vcvtp_s64_v:
6549 case NEON::BI__builtin_neon_vcvtpq_s64_v:
6550 case NEON::BI__builtin_neon_vcvtp_u64_v:
6551 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
6552 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
6553 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(CGF: this, IntTypeFlags: Type) };
6554 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vcvtp");
6555 }
6556 case NEON::BI__builtin_neon_vmulx_v:
6557 case NEON::BI__builtin_neon_vmulxq_v: {
6558 Int = Intrinsic::aarch64_neon_fmulx;
6559 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vmulx");
6560 }
6561 case NEON::BI__builtin_neon_vmulxh_lane_f16:
6562 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
6563 // vmulx_lane should be mapped to Neon scalar mulx after
6564 // extracting the scalar element
6565 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2], Name: "extract");
6566 Ops.pop_back();
6567 Int = Intrinsic::aarch64_neon_fmulx;
6568 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vmulx");
6569 }
6570 case NEON::BI__builtin_neon_vmul_lane_v:
6571 case NEON::BI__builtin_neon_vmul_laneq_v: {
6572 // v1f64 vmul_lane should be mapped to Neon scalar mul lane
6573 bool Quad = false;
6574 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
6575 Quad = true;
6576 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
6577 llvm::FixedVectorType *VTy =
6578 GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
6579 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: VTy);
6580 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2], Name: "extract");
6581 Value *Result = Builder.CreateFMul(L: Ops[0], R: Ops[1]);
6582 return Builder.CreateBitCast(V: Result, DestTy: Ty);
6583 }
6584 case NEON::BI__builtin_neon_vpmaxnm_v:
6585 case NEON::BI__builtin_neon_vpmaxnmq_v: {
6586 Int = Intrinsic::aarch64_neon_fmaxnmp;
6587 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpmaxnm");
6588 }
6589 case NEON::BI__builtin_neon_vpminnm_v:
6590 case NEON::BI__builtin_neon_vpminnmq_v: {
6591 Int = Intrinsic::aarch64_neon_fminnmp;
6592 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vpminnm");
6593 }
6594 case NEON::BI__builtin_neon_vsqrth_f16: {
6595 Int = Builder.getIsFPConstrained()
6596 ? Intrinsic::experimental_constrained_sqrt
6597 : Intrinsic::sqrt;
6598 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: HalfTy), Ops, name: "vsqrt");
6599 }
6600 case NEON::BI__builtin_neon_vsqrt_v:
6601 case NEON::BI__builtin_neon_vsqrtq_v: {
6602 Int = Builder.getIsFPConstrained()
6603 ? Intrinsic::experimental_constrained_sqrt
6604 : Intrinsic::sqrt;
6605 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6606 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vsqrt");
6607 }
6608 case NEON::BI__builtin_neon_vrbit_v:
6609 case NEON::BI__builtin_neon_vrbitq_v: {
6610 Int = Intrinsic::bitreverse;
6611 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vrbit");
6612 }
6613 case NEON::BI__builtin_neon_vmaxv_f16: {
6614 Int = Intrinsic::aarch64_neon_fmaxv;
6615 Ty = HalfTy;
6616 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6617 llvm::Type *Tys[2] = {Ty, VTy};
6618 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxv");
6619 }
6620 case NEON::BI__builtin_neon_vmaxvq_f16: {
6621 Int = Intrinsic::aarch64_neon_fmaxv;
6622 Ty = HalfTy;
6623 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6624 llvm::Type *Tys[2] = {Ty, VTy};
6625 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxv");
6626 }
6627 case NEON::BI__builtin_neon_vminv_f16: {
6628 Int = Intrinsic::aarch64_neon_fminv;
6629 Ty = HalfTy;
6630 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6631 llvm::Type *Tys[2] = {Ty, VTy};
6632 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminv");
6633 }
6634 case NEON::BI__builtin_neon_vminvq_f16: {
6635 Int = Intrinsic::aarch64_neon_fminv;
6636 Ty = HalfTy;
6637 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6638 llvm::Type *Tys[2] = {Ty, VTy};
6639 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminv");
6640 }
6641 case NEON::BI__builtin_neon_vmaxnmv_f16: {
6642 Int = Intrinsic::aarch64_neon_fmaxnmv;
6643 Ty = HalfTy;
6644 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6645 llvm::Type *Tys[2] = {Ty, VTy};
6646 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxnmv");
6647 }
6648 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
6649 Int = Intrinsic::aarch64_neon_fmaxnmv;
6650 Ty = HalfTy;
6651 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6652 llvm::Type *Tys[2] = {Ty, VTy};
6653 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vmaxnmv");
6654 }
6655 case NEON::BI__builtin_neon_vminnmv_f16: {
6656 Int = Intrinsic::aarch64_neon_fminnmv;
6657 Ty = HalfTy;
6658 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4);
6659 llvm::Type *Tys[2] = {Ty, VTy};
6660 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminnmv");
6661 }
6662 case NEON::BI__builtin_neon_vminnmvq_f16: {
6663 Int = Intrinsic::aarch64_neon_fminnmv;
6664 Ty = HalfTy;
6665 VTy = llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8);
6666 llvm::Type *Tys[2] = {Ty, VTy};
6667 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vminnmv");
6668 }
6669 case NEON::BI__builtin_neon_vmul_n_f64: {
6670 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: DoubleTy);
6671 Value *RHS = Builder.CreateBitCast(V: Ops[1], DestTy: DoubleTy);
6672 return Builder.CreateFMul(L: Ops[0], R: RHS);
6673 }
6674 case NEON::BI__builtin_neon_vaddlv_u8:
6675 case NEON::BI__builtin_neon_vaddlvq_u8:
6676 case NEON::BI__builtin_neon_vaddlv_u16:
6677 case NEON::BI__builtin_neon_vaddlvq_u16: {
6678 Int = Intrinsic::aarch64_neon_uaddlv;
6679 Ty = Int32Ty;
6680 VTy = cast<llvm::FixedVectorType>(Val: Ops[0]->getType());
6681 llvm::Type *Tys[2] = {Ty, VTy};
6682 Value *Result = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vaddlv");
6683 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6684 return Builder.CreateTrunc(V: Result, DestTy: Int16Ty);
6685 return Result;
6686 }
6687 case NEON::BI__builtin_neon_vaddlv_s8:
6688 case NEON::BI__builtin_neon_vaddlvq_s8:
6689 case NEON::BI__builtin_neon_vaddlv_s16:
6690 case NEON::BI__builtin_neon_vaddlvq_s16: {
6691 Int = Intrinsic::aarch64_neon_saddlv;
6692 Ty = Int32Ty;
6693 VTy = cast<llvm::FixedVectorType>(Val: Ops[0]->getType());
6694 llvm::Type *Tys[2] = {Ty, VTy};
6695 Value *Result = EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vaddlv");
6696 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6697 return Builder.CreateTrunc(V: Result, DestTy: Int16Ty);
6698 return Result;
6699 }
6700 case NEON::BI__builtin_neon_vsri_n_v:
6701 case NEON::BI__builtin_neon_vsriq_n_v: {
6702 Int = Intrinsic::aarch64_neon_vsri;
6703 llvm::Function *Intrin = CGM.getIntrinsic(IID: Int, Tys: Ty);
6704 return EmitNeonCall(F: Intrin, Ops, name: "vsri_n");
6705 }
6706 case NEON::BI__builtin_neon_vsli_n_v:
6707 case NEON::BI__builtin_neon_vsliq_n_v: {
6708 Int = Intrinsic::aarch64_neon_vsli;
6709 llvm::Function *Intrin = CGM.getIntrinsic(IID: Int, Tys: Ty);
6710 return EmitNeonCall(F: Intrin, Ops, name: "vsli_n");
6711 }
6712 case NEON::BI__builtin_neon_vsra_n_v:
6713 case NEON::BI__builtin_neon_vsraq_n_v:
6714 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: Ty);
6715 Ops[1] = EmitNeonRShiftImm(Vec: Ops[1], Shift: Ops[2], Ty, usgn, name: "vsra_n");
6716 return Builder.CreateAdd(LHS: Ops[0], RHS: Ops[1]);
6717 case NEON::BI__builtin_neon_vrsra_n_v:
6718 case NEON::BI__builtin_neon_vrsraq_n_v: {
6719 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
6720 SmallVector<llvm::Value*,2> TmpOps;
6721 TmpOps.push_back(Elt: Ops[1]);
6722 TmpOps.push_back(Elt: Ops[2]);
6723 Function* F = CGM.getIntrinsic(IID: Int, Tys: Ty);
6724 llvm::Value *tmp = EmitNeonCall(F, Ops&: TmpOps, name: "vrshr_n", shift: 1, rightshift: true);
6725 Ops[0] = Builder.CreateBitCast(V: Ops[0], DestTy: VTy);
6726 return Builder.CreateAdd(LHS: Ops[0], RHS: tmp);
6727 }
6728 case NEON::BI__builtin_neon_vld1_v:
6729 case NEON::BI__builtin_neon_vld1q_v: {
6730 return Builder.CreateAlignedLoad(Ty: VTy, Addr: Ops[0], Align: PtrOp0.getAlignment());
6731 }
6732 case NEON::BI__builtin_neon_vst1_v:
6733 case NEON::BI__builtin_neon_vst1q_v:
6734 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: VTy);
6735 return Builder.CreateAlignedStore(Val: Ops[1], Addr: Ops[0], Align: PtrOp0.getAlignment());
6736 case NEON::BI__builtin_neon_vld1_lane_v:
6737 case NEON::BI__builtin_neon_vld1q_lane_v: {
6738 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6739 Ops[0] = Builder.CreateAlignedLoad(Ty: VTy->getElementType(), Addr: Ops[0],
6740 Align: PtrOp0.getAlignment());
6741 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vld1_lane");
6742 }
6743 case NEON::BI__builtin_neon_vldap1_lane_s64:
6744 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
6745 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6746 llvm::LoadInst *LI = Builder.CreateAlignedLoad(
6747 Ty: VTy->getElementType(), Addr: Ops[0], Align: PtrOp0.getAlignment());
6748 LI->setAtomic(Ordering: llvm::AtomicOrdering::Acquire);
6749 Ops[0] = LI;
6750 return Builder.CreateInsertElement(Vec: Ops[1], NewElt: Ops[0], Idx: Ops[2], Name: "vldap1_lane");
6751 }
6752 case NEON::BI__builtin_neon_vld1_dup_v:
6753 case NEON::BI__builtin_neon_vld1q_dup_v: {
6754 Value *V = PoisonValue::get(T: Ty);
6755 Ops[0] = Builder.CreateAlignedLoad(Ty: VTy->getElementType(), Addr: Ops[0],
6756 Align: PtrOp0.getAlignment());
6757 llvm::Constant *CI = ConstantInt::get(Ty: Int32Ty, V: 0);
6758 Ops[0] = Builder.CreateInsertElement(Vec: V, NewElt: Ops[0], Idx: CI);
6759 return EmitNeonSplat(V: Ops[0], C: CI);
6760 }
6761 case NEON::BI__builtin_neon_vst1_lane_v:
6762 case NEON::BI__builtin_neon_vst1q_lane_v:
6763 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6764 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2]);
6765 return Builder.CreateAlignedStore(Val: Ops[1], Addr: Ops[0], Align: PtrOp0.getAlignment());
6766 case NEON::BI__builtin_neon_vstl1_lane_s64:
6767 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
6768 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6769 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: Ops[2]);
6770 llvm::StoreInst *SI =
6771 Builder.CreateAlignedStore(Val: Ops[1], Addr: Ops[0], Align: PtrOp0.getAlignment());
6772 SI->setAtomic(Ordering: llvm::AtomicOrdering::Release);
6773 return SI;
6774 }
6775 case NEON::BI__builtin_neon_vld2_v:
6776 case NEON::BI__builtin_neon_vld2q_v: {
6777 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6778 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld2, Tys);
6779 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld2");
6780 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6781 }
6782 case NEON::BI__builtin_neon_vld3_v:
6783 case NEON::BI__builtin_neon_vld3q_v: {
6784 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6785 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld3, Tys);
6786 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld3");
6787 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6788 }
6789 case NEON::BI__builtin_neon_vld4_v:
6790 case NEON::BI__builtin_neon_vld4q_v: {
6791 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6792 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld4, Tys);
6793 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld4");
6794 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6795 }
6796 case NEON::BI__builtin_neon_vld2_dup_v:
6797 case NEON::BI__builtin_neon_vld2q_dup_v: {
6798 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6799 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld2r, Tys);
6800 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld2");
6801 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6802 }
6803 case NEON::BI__builtin_neon_vld3_dup_v:
6804 case NEON::BI__builtin_neon_vld3q_dup_v: {
6805 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6806 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld3r, Tys);
6807 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld3");
6808 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6809 }
6810 case NEON::BI__builtin_neon_vld4_dup_v:
6811 case NEON::BI__builtin_neon_vld4q_dup_v: {
6812 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6813 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld4r, Tys);
6814 Ops[1] = Builder.CreateCall(Callee: F, Args: Ops[1], Name: "vld4");
6815 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6816 }
6817 case NEON::BI__builtin_neon_vld2_lane_v:
6818 case NEON::BI__builtin_neon_vld2q_lane_v: {
6819 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6820 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld2lane, Tys);
6821 std::rotate(first: Ops.begin() + 1, middle: Ops.begin() + 2, last: Ops.end());
6822 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6823 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6824 Ops[3] = Builder.CreateZExt(V: Ops[3], DestTy: Int64Ty);
6825 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: "vld2_lane");
6826 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6827 }
6828 case NEON::BI__builtin_neon_vld3_lane_v:
6829 case NEON::BI__builtin_neon_vld3q_lane_v: {
6830 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6831 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld3lane, Tys);
6832 std::rotate(first: Ops.begin() + 1, middle: Ops.begin() + 2, last: Ops.end());
6833 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6834 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6835 Ops[3] = Builder.CreateBitCast(V: Ops[3], DestTy: Ty);
6836 Ops[4] = Builder.CreateZExt(V: Ops[4], DestTy: Int64Ty);
6837 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: "vld3_lane");
6838 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6839 }
6840 case NEON::BI__builtin_neon_vld4_lane_v:
6841 case NEON::BI__builtin_neon_vld4q_lane_v: {
6842 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6843 Function *F = CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_ld4lane, Tys);
6844 std::rotate(first: Ops.begin() + 1, middle: Ops.begin() + 2, last: Ops.end());
6845 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6846 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6847 Ops[3] = Builder.CreateBitCast(V: Ops[3], DestTy: Ty);
6848 Ops[4] = Builder.CreateBitCast(V: Ops[4], DestTy: Ty);
6849 Ops[5] = Builder.CreateZExt(V: Ops[5], DestTy: Int64Ty);
6850 Ops[1] = Builder.CreateCall(Callee: F, Args: ArrayRef(Ops).slice(N: 1), Name: "vld4_lane");
6851 return Builder.CreateDefaultAlignedStore(Val: Ops[1], Addr: Ops[0]);
6852 }
6853 case NEON::BI__builtin_neon_vst2_v:
6854 case NEON::BI__builtin_neon_vst2q_v: {
6855 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6856 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
6857 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st2, Tys),
6858 Ops, name: "");
6859 }
6860 case NEON::BI__builtin_neon_vst2_lane_v:
6861 case NEON::BI__builtin_neon_vst2q_lane_v: {
6862 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6863 Ops[2] = Builder.CreateZExt(V: Ops[2], DestTy: Int64Ty);
6864 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6865 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st2lane, Tys),
6866 Ops, name: "");
6867 }
6868 case NEON::BI__builtin_neon_vst3_v:
6869 case NEON::BI__builtin_neon_vst3q_v: {
6870 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6871 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6872 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st3, Tys),
6873 Ops, name: "");
6874 }
6875 case NEON::BI__builtin_neon_vst3_lane_v:
6876 case NEON::BI__builtin_neon_vst3q_lane_v: {
6877 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6878 Ops[3] = Builder.CreateZExt(V: Ops[3], DestTy: Int64Ty);
6879 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6880 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st3lane, Tys),
6881 Ops, name: "");
6882 }
6883 case NEON::BI__builtin_neon_vst4_v:
6884 case NEON::BI__builtin_neon_vst4q_v: {
6885 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6886 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6887 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st4, Tys),
6888 Ops, name: "");
6889 }
6890 case NEON::BI__builtin_neon_vst4_lane_v:
6891 case NEON::BI__builtin_neon_vst4q_lane_v: {
6892 std::rotate(first: Ops.begin(), middle: Ops.begin() + 1, last: Ops.end());
6893 Ops[4] = Builder.CreateZExt(V: Ops[4], DestTy: Int64Ty);
6894 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
6895 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_st4lane, Tys),
6896 Ops, name: "");
6897 }
6898 case NEON::BI__builtin_neon_vtrn_v:
6899 case NEON::BI__builtin_neon_vtrnq_v: {
6900 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6901 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6902 Value *SV = nullptr;
6903
6904 for (unsigned vi = 0; vi != 2; ++vi) {
6905 SmallVector<int, 16> Indices;
6906 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6907 Indices.push_back(Elt: i+vi);
6908 Indices.push_back(Elt: i+e+vi);
6909 }
6910 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
6911 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vtrn");
6912 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
6913 }
6914 return SV;
6915 }
6916 case NEON::BI__builtin_neon_vuzp_v:
6917 case NEON::BI__builtin_neon_vuzpq_v: {
6918 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6919 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6920 Value *SV = nullptr;
6921
6922 for (unsigned vi = 0; vi != 2; ++vi) {
6923 SmallVector<int, 16> Indices;
6924 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
6925 Indices.push_back(Elt: 2*i+vi);
6926
6927 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
6928 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vuzp");
6929 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
6930 }
6931 return SV;
6932 }
6933 case NEON::BI__builtin_neon_vzip_v:
6934 case NEON::BI__builtin_neon_vzipq_v: {
6935 Ops[1] = Builder.CreateBitCast(V: Ops[1], DestTy: Ty);
6936 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: Ty);
6937 Value *SV = nullptr;
6938
6939 for (unsigned vi = 0; vi != 2; ++vi) {
6940 SmallVector<int, 16> Indices;
6941 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6942 Indices.push_back(Elt: (i + vi*e) >> 1);
6943 Indices.push_back(Elt: ((i + vi*e) >> 1)+e);
6944 }
6945 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ptr: Ops[0], Idx0: vi);
6946 SV = Builder.CreateShuffleVector(V1: Ops[1], V2: Ops[2], Mask: Indices, Name: "vzip");
6947 SV = Builder.CreateDefaultAlignedStore(Val: SV, Addr);
6948 }
6949 return SV;
6950 }
6951 case NEON::BI__builtin_neon_vqtbl1q_v: {
6952 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl1, Tys: Ty),
6953 Ops, name: "vtbl1");
6954 }
6955 case NEON::BI__builtin_neon_vqtbl2q_v: {
6956 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl2, Tys: Ty),
6957 Ops, name: "vtbl2");
6958 }
6959 case NEON::BI__builtin_neon_vqtbl3q_v: {
6960 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl3, Tys: Ty),
6961 Ops, name: "vtbl3");
6962 }
6963 case NEON::BI__builtin_neon_vqtbl4q_v: {
6964 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbl4, Tys: Ty),
6965 Ops, name: "vtbl4");
6966 }
6967 case NEON::BI__builtin_neon_vqtbx1q_v: {
6968 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx1, Tys: Ty),
6969 Ops, name: "vtbx1");
6970 }
6971 case NEON::BI__builtin_neon_vqtbx2q_v: {
6972 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx2, Tys: Ty),
6973 Ops, name: "vtbx2");
6974 }
6975 case NEON::BI__builtin_neon_vqtbx3q_v: {
6976 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx3, Tys: Ty),
6977 Ops, name: "vtbx3");
6978 }
6979 case NEON::BI__builtin_neon_vqtbx4q_v: {
6980 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_tbx4, Tys: Ty),
6981 Ops, name: "vtbx4");
6982 }
6983 case NEON::BI__builtin_neon_vsqadd_v:
6984 case NEON::BI__builtin_neon_vsqaddq_v: {
6985 Int = Intrinsic::aarch64_neon_usqadd;
6986 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vsqadd");
6987 }
6988 case NEON::BI__builtin_neon_vuqadd_v:
6989 case NEON::BI__builtin_neon_vuqaddq_v: {
6990 Int = Intrinsic::aarch64_neon_suqadd;
6991 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vuqadd");
6992 }
6993
6994 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
6995 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
6996 case NEON::BI__builtin_neon_vluti2_laneq_f16:
6997 case NEON::BI__builtin_neon_vluti2_laneq_p16:
6998 case NEON::BI__builtin_neon_vluti2_laneq_p8:
6999 case NEON::BI__builtin_neon_vluti2_laneq_s16:
7000 case NEON::BI__builtin_neon_vluti2_laneq_s8:
7001 case NEON::BI__builtin_neon_vluti2_laneq_u16:
7002 case NEON::BI__builtin_neon_vluti2_laneq_u8: {
7003 Int = Intrinsic::aarch64_neon_vluti2_laneq;
7004 llvm::Type *Tys[2];
7005 Tys[0] = Ty;
7006 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
7007 /*isQuad*/ false));
7008 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_laneq");
7009 }
7010 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
7011 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
7012 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
7013 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
7014 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
7015 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
7016 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
7017 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
7018 case NEON::BI__builtin_neon_vluti2q_laneq_u8: {
7019 Int = Intrinsic::aarch64_neon_vluti2_laneq;
7020 llvm::Type *Tys[2];
7021 Tys[0] = Ty;
7022 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
7023 /*isQuad*/ true));
7024 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_laneq");
7025 }
7026 case NEON::BI__builtin_neon_vluti2_lane_mf8:
7027 case NEON::BI__builtin_neon_vluti2_lane_bf16:
7028 case NEON::BI__builtin_neon_vluti2_lane_f16:
7029 case NEON::BI__builtin_neon_vluti2_lane_p16:
7030 case NEON::BI__builtin_neon_vluti2_lane_p8:
7031 case NEON::BI__builtin_neon_vluti2_lane_s16:
7032 case NEON::BI__builtin_neon_vluti2_lane_s8:
7033 case NEON::BI__builtin_neon_vluti2_lane_u16:
7034 case NEON::BI__builtin_neon_vluti2_lane_u8: {
7035 Int = Intrinsic::aarch64_neon_vluti2_lane;
7036 llvm::Type *Tys[2];
7037 Tys[0] = Ty;
7038 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
7039 /*isQuad*/ false));
7040 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_lane");
7041 }
7042 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
7043 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
7044 case NEON::BI__builtin_neon_vluti2q_lane_f16:
7045 case NEON::BI__builtin_neon_vluti2q_lane_p16:
7046 case NEON::BI__builtin_neon_vluti2q_lane_p8:
7047 case NEON::BI__builtin_neon_vluti2q_lane_s16:
7048 case NEON::BI__builtin_neon_vluti2q_lane_s8:
7049 case NEON::BI__builtin_neon_vluti2q_lane_u16:
7050 case NEON::BI__builtin_neon_vluti2q_lane_u8: {
7051 Int = Intrinsic::aarch64_neon_vluti2_lane;
7052 llvm::Type *Tys[2];
7053 Tys[0] = Ty;
7054 Tys[1] = GetNeonType(CGF: this, TypeFlags: NeonTypeFlags(Type.getEltType(), false,
7055 /*isQuad*/ true));
7056 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys), Ops, name: "vluti2_lane");
7057 }
7058 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
7059 case NEON::BI__builtin_neon_vluti4q_lane_p8:
7060 case NEON::BI__builtin_neon_vluti4q_lane_s8:
7061 case NEON::BI__builtin_neon_vluti4q_lane_u8: {
7062 Int = Intrinsic::aarch64_neon_vluti4q_lane;
7063 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_lane");
7064 }
7065 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
7066 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
7067 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
7068 case NEON::BI__builtin_neon_vluti4q_laneq_u8: {
7069 Int = Intrinsic::aarch64_neon_vluti4q_laneq;
7070 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_laneq");
7071 }
7072 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
7073 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
7074 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
7075 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
7076 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: {
7077 Int = Intrinsic::aarch64_neon_vluti4q_lane_x2;
7078 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_lane_x2");
7079 }
7080 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
7081 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
7082 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
7083 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
7084 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: {
7085 Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
7086 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "vluti4q_laneq_x2");
7087 }
7088 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
7089 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fmmla,
7090 Tys: {llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8),
7091 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16)},
7092 Ops, E, name: "fmmla");
7093 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
7094 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fmmla,
7095 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4),
7096 llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16)},
7097 Ops, E, name: "fmmla");
7098 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
7099 ExtractLow = true;
7100 [[fallthrough]];
7101 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
7102 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
7103 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl1,
7104 Ty0: llvm::FixedVectorType::get(ElementType: BFloatTy, NumElts: 8),
7105 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt1");
7106 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
7107 ExtractLow = true;
7108 [[fallthrough]];
7109 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
7110 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
7111 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl2,
7112 Ty0: llvm::FixedVectorType::get(ElementType: BFloatTy, NumElts: 8),
7113 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt2");
7114 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
7115 ExtractLow = true;
7116 [[fallthrough]];
7117 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
7118 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
7119 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl1,
7120 Ty0: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8),
7121 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt1");
7122 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
7123 ExtractLow = true;
7124 [[fallthrough]];
7125 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
7126 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
7127 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_cvtl2,
7128 Ty0: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8),
7129 Ty1: Ops[0]->getType(), Extract: ExtractLow, Ops, E, name: "vbfcvt2");
7130 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
7131 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn,
7132 Ty0: llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 8),
7133 Ty1: Ops[0]->getType(), Extract: false, Ops, E, name: "vfcvtn");
7134 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
7135 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn,
7136 Ty0: llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 8),
7137 Ty1: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 4), Extract: false, Ops,
7138 E, name: "vfcvtn");
7139 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
7140 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn,
7141 Ty0: llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16),
7142 Ty1: llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8), Extract: false, Ops,
7143 E, name: "vfcvtn");
7144 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: {
7145 llvm::Type *Ty = llvm::FixedVectorType::get(ElementType: Int8Ty, NumElts: 16);
7146 Ops[0] = Builder.CreateInsertVector(DstType: Ty, SrcVec: PoisonValue::get(T: Ty), SubVec: Ops[0],
7147 Idx: uint64_t(0));
7148 return EmitFP8NeonCvtCall(IID: Intrinsic::aarch64_neon_fp8_fcvtn2, Ty0: Ty,
7149 Ty1: Ops[1]->getType(), Extract: false, Ops, E, name: "vfcvtn2");
7150 }
7151
7152 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
7153 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
7154 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot2, ExtendLaneArg: false, RetTy: HalfTy,
7155 Ops, E, name: "fdot2");
7156 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
7157 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
7158 ExtendLaneArg = true;
7159 [[fallthrough]];
7160 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
7161 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
7162 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot2_lane,
7163 ExtendLaneArg, RetTy: HalfTy, Ops, E, name: "fdot2_lane");
7164 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
7165 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
7166 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot4, ExtendLaneArg: false,
7167 RetTy: FloatTy, Ops, E, name: "fdot4");
7168 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
7169 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
7170 ExtendLaneArg = true;
7171 [[fallthrough]];
7172 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
7173 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
7174 return EmitFP8NeonFDOTCall(IID: Intrinsic::aarch64_neon_fp8_fdot4_lane,
7175 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "fdot4_lane");
7176
7177 case NEON::BI__builtin_neon_vdot_f32_f16:
7178 case NEON::BI__builtin_neon_vdotq_f32_f16: {
7179 llvm::Type *InputTy =
7180 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
7181 llvm::Type *Tys[2] = {Ty, InputTy};
7182 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_fdot, Tys),
7183 Ops, name: "vdot");
7184 }
7185
7186 case NEON::BI__builtin_neon_vdot_lane_f32_f16:
7187 case NEON::BI__builtin_neon_vdot_laneq_f32_f16:
7188 case NEON::BI__builtin_neon_vdotq_lane_f32_f16:
7189 case NEON::BI__builtin_neon_vdotq_laneq_f32_f16: {
7190 llvm::FixedVectorType *InputTy =
7191 llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: Ty->getPrimitiveSizeInBits() / 16);
7192 llvm::FixedVectorType *LaneTy = llvm::FixedVectorType::get(
7193 ElementType: HalfTy, NumElts: Ops[2]->getType()->getPrimitiveSizeInBits() / 16);
7194 // Treat the lane argument as a splat and use non-lane version of the
7195 // intrinsic.
7196 Ops[2] = Builder.CreateBitCast(V: Ops[2], DestTy: LaneTy);
7197 Ops[2] = EmitNeonSplat(V: Ops[2], C: cast<ConstantInt>(Val: Ops[3]),
7198 Count: InputTy->getElementCount());
7199 llvm::Type *Tys[2] = {Ty, InputTy};
7200 Ops.pop_back();
7201 return EmitNeonCall(F: CGM.getIntrinsic(IID: Intrinsic::aarch64_neon_fdot, Tys),
7202 Ops, name: "vdot");
7203 }
7204
7205 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
7206 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalb,
7207 Tys: {llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8)}, Ops, E,
7208 name: "vmlal");
7209 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
7210 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalt,
7211 Tys: {llvm::FixedVectorType::get(ElementType: HalfTy, NumElts: 8)}, Ops, E,
7212 name: "vmlal");
7213 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
7214 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlallbb,
7215 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7216 name: "vmlall");
7217 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
7218 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlallbt,
7219 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7220 name: "vmlall");
7221 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
7222 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalltb,
7223 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7224 name: "vmlall");
7225 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
7226 return EmitFP8NeonCall(IID: Intrinsic::aarch64_neon_fp8_fmlalltt,
7227 Tys: {llvm::FixedVectorType::get(ElementType: FloatTy, NumElts: 4)}, Ops, E,
7228 name: "vmlall");
7229 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
7230 ExtendLaneArg = true;
7231 [[fallthrough]];
7232 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
7233 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalb_lane,
7234 ExtendLaneArg, RetTy: HalfTy, Ops, E, name: "vmlal_lane");
7235 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
7236 ExtendLaneArg = true;
7237 [[fallthrough]];
7238 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
7239 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalt_lane,
7240 ExtendLaneArg, RetTy: HalfTy, Ops, E, name: "vmlal_lane");
7241 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
7242 ExtendLaneArg = true;
7243 [[fallthrough]];
7244 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
7245 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlallbb_lane,
7246 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7247 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
7248 ExtendLaneArg = true;
7249 [[fallthrough]];
7250 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
7251 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlallbt_lane,
7252 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7253 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
7254 ExtendLaneArg = true;
7255 [[fallthrough]];
7256 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
7257 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalltb_lane,
7258 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7259 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
7260 ExtendLaneArg = true;
7261 [[fallthrough]];
7262 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
7263 return EmitFP8NeonFMLACall(IID: Intrinsic::aarch64_neon_fp8_fmlalltt_lane,
7264 ExtendLaneArg, RetTy: FloatTy, Ops, E, name: "vmlall_lane");
7265 case NEON::BI__builtin_neon_vamin_f16:
7266 case NEON::BI__builtin_neon_vaminq_f16:
7267 case NEON::BI__builtin_neon_vamin_f32:
7268 case NEON::BI__builtin_neon_vaminq_f32:
7269 case NEON::BI__builtin_neon_vaminq_f64: {
7270 Int = Intrinsic::aarch64_neon_famin;
7271 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "famin");
7272 }
7273 case NEON::BI__builtin_neon_vamax_f16:
7274 case NEON::BI__builtin_neon_vamaxq_f16:
7275 case NEON::BI__builtin_neon_vamax_f32:
7276 case NEON::BI__builtin_neon_vamaxq_f32:
7277 case NEON::BI__builtin_neon_vamaxq_f64: {
7278 Int = Intrinsic::aarch64_neon_famax;
7279 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "famax");
7280 }
7281 case NEON::BI__builtin_neon_vscale_f16:
7282 case NEON::BI__builtin_neon_vscaleq_f16:
7283 case NEON::BI__builtin_neon_vscale_f32:
7284 case NEON::BI__builtin_neon_vscaleq_f32:
7285 case NEON::BI__builtin_neon_vscaleq_f64: {
7286 Int = Intrinsic::aarch64_neon_fp8_fscale;
7287 return EmitNeonCall(F: CGM.getIntrinsic(IID: Int, Tys: Ty), Ops, name: "fscale");
7288 }
7289 }
7290}
7291
7292Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
7293 const CallExpr *E) {
7294 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
7295 BuiltinID == BPF::BI__builtin_btf_type_id ||
7296 BuiltinID == BPF::BI__builtin_preserve_type_info ||
7297 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
7298 "unexpected BPF builtin");
7299
7300 // A sequence number, injected into IR builtin functions, to
7301 // prevent CSE given the only difference of the function
7302 // may just be the debuginfo metadata.
7303 static uint32_t BuiltinSeqNum;
7304
7305 switch (BuiltinID) {
7306 default:
7307 llvm_unreachable("Unexpected BPF builtin");
7308 case BPF::BI__builtin_preserve_field_info: {
7309 const Expr *Arg = E->getArg(Arg: 0);
7310 bool IsBitField = Arg->IgnoreParens()->getObjectKind() == OK_BitField;
7311
7312 if (!getDebugInfo()) {
7313 CGM.Error(loc: E->getExprLoc(),
7314 error: "using __builtin_preserve_field_info() without -g");
7315 return IsBitField ? EmitLValue(E: Arg).getRawBitFieldPointer(CGF&: *this)
7316 : EmitLValue(E: Arg).emitRawPointer(CGF&: *this);
7317 }
7318
7319 // Enable underlying preserve_*_access_index() generation. Field info
7320 // never dereferences the field, so skip preserve_static_offset markers:
7321 // the backend would replace the access indices under them with plain GEPs.
7322 bool OldIsInPreservedAIRegion = IsInPreservedAIRegion;
7323 bool OldIsInPreserveFieldInfo = IsInPreserveFieldInfo;
7324 IsInPreservedAIRegion = true;
7325 IsInPreserveFieldInfo = true;
7326 Value *FieldAddr = IsBitField ? EmitLValue(E: Arg).getRawBitFieldPointer(CGF&: *this)
7327 : EmitLValue(E: Arg).emitRawPointer(CGF&: *this);
7328 IsInPreservedAIRegion = OldIsInPreservedAIRegion;
7329 IsInPreserveFieldInfo = OldIsInPreserveFieldInfo;
7330
7331 ConstantInt *C = cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 1)));
7332 Value *InfoKind = ConstantInt::get(Ty: Int64Ty, V: C->getSExtValue());
7333
7334 // Built the IR for the preserve_field_info intrinsic.
7335 llvm::Function *FnGetFieldInfo = Intrinsic::getOrInsertDeclaration(
7336 M: &CGM.getModule(), id: Intrinsic::bpf_preserve_field_info,
7337 OverloadTys: {FieldAddr->getType()});
7338 return Builder.CreateCall(Callee: FnGetFieldInfo, Args: {FieldAddr, InfoKind});
7339 }
7340 case BPF::BI__builtin_btf_type_id:
7341 case BPF::BI__builtin_preserve_type_info: {
7342 if (!getDebugInfo()) {
7343 CGM.Error(loc: E->getExprLoc(), error: "using builtin function without -g");
7344 return nullptr;
7345 }
7346
7347 const Expr *Arg0 = E->getArg(Arg: 0);
7348 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
7349 Ty: Arg0->getType(), Loc: Arg0->getExprLoc());
7350
7351 ConstantInt *Flag = cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 1)));
7352 Value *FlagValue = ConstantInt::get(Ty: Int64Ty, V: Flag->getSExtValue());
7353 Value *SeqNumVal = ConstantInt::get(Ty: Int32Ty, V: BuiltinSeqNum++);
7354
7355 llvm::Function *FnDecl;
7356 if (BuiltinID == BPF::BI__builtin_btf_type_id)
7357 FnDecl = Intrinsic::getOrInsertDeclaration(
7358 M: &CGM.getModule(), id: Intrinsic::bpf_btf_type_id, OverloadTys: {});
7359 else
7360 FnDecl = Intrinsic::getOrInsertDeclaration(
7361 M: &CGM.getModule(), id: Intrinsic::bpf_preserve_type_info, OverloadTys: {});
7362 CallInst *Fn = Builder.CreateCall(Callee: FnDecl, Args: {SeqNumVal, FlagValue});
7363 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
7364 return Fn;
7365 }
7366 case BPF::BI__builtin_preserve_enum_value: {
7367 if (!getDebugInfo()) {
7368 CGM.Error(loc: E->getExprLoc(), error: "using builtin function without -g");
7369 return nullptr;
7370 }
7371
7372 const Expr *Arg0 = E->getArg(Arg: 0);
7373 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
7374 Ty: Arg0->getType(), Loc: Arg0->getExprLoc());
7375
7376 // Find enumerator
7377 const auto *UO = cast<UnaryOperator>(Val: Arg0->IgnoreParens());
7378 const auto *CE = cast<CStyleCastExpr>(Val: UO->getSubExpr());
7379 const auto *DR = cast<DeclRefExpr>(Val: CE->getSubExpr());
7380 const auto *Enumerator = cast<EnumConstantDecl>(Val: DR->getDecl());
7381
7382 auto InitVal = Enumerator->getInitVal();
7383 std::string InitValStr;
7384 if (InitVal.isNegative() || InitVal > uint64_t(INT64_MAX))
7385 InitValStr = std::to_string(val: InitVal.getSExtValue());
7386 else
7387 InitValStr = std::to_string(val: InitVal.getZExtValue());
7388 std::string EnumStr = Enumerator->getNameAsString() + ":" + InitValStr;
7389 Value *EnumStrVal = Builder.CreateGlobalString(Str: EnumStr);
7390
7391 ConstantInt *Flag = cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 1)));
7392 Value *FlagValue = ConstantInt::get(Ty: Int64Ty, V: Flag->getSExtValue());
7393 Value *SeqNumVal = ConstantInt::get(Ty: Int32Ty, V: BuiltinSeqNum++);
7394
7395 llvm::Function *IntrinsicFn = Intrinsic::getOrInsertDeclaration(
7396 M: &CGM.getModule(), id: Intrinsic::bpf_preserve_enum_value, OverloadTys: {});
7397 CallInst *Fn =
7398 Builder.CreateCall(Callee: IntrinsicFn, Args: {SeqNumVal, EnumStrVal, FlagValue});
7399 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
7400 return Fn;
7401 }
7402 }
7403}
7404
7405llvm::Value *CodeGenFunction::
7406BuildVector(ArrayRef<llvm::Value*> Ops) {
7407 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
7408 "Not a power-of-two sized vector!");
7409 bool AllConstants = true;
7410 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
7411 AllConstants &= isa<Constant>(Val: Ops[i]);
7412
7413 // If this is a constant vector, create a ConstantVector.
7414 if (AllConstants) {
7415 SmallVector<llvm::Constant*, 16> CstOps;
7416 for (llvm::Value *Op : Ops)
7417 CstOps.push_back(Elt: cast<Constant>(Val: Op));
7418 return llvm::ConstantVector::get(V: CstOps);
7419 }
7420
7421 // Otherwise, insertelement the values to build the vector.
7422 Value *Result = llvm::PoisonValue::get(
7423 T: llvm::FixedVectorType::get(ElementType: Ops[0]->getType(), NumElts: Ops.size()));
7424
7425 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
7426 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Ops[i], Idx: Builder.getInt64(C: i));
7427
7428 return Result;
7429}
7430
7431Value *CodeGenFunction::EmitAArch64CpuInit() {
7432 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
7433 llvm::FunctionCallee Func =
7434 CGM.CreateRuntimeFunction(Ty: FTy, Name: "__init_cpu_features_resolver");
7435 cast<llvm::GlobalValue>(Val: Func.getCallee())->setDSOLocal(true);
7436 cast<llvm::GlobalValue>(Val: Func.getCallee())
7437 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
7438 return Builder.CreateCall(Callee: Func);
7439}
7440
7441Value *CodeGenFunction::EmitAArch64CpuSupports(const CallExpr *E) {
7442 const Expr *ArgExpr = E->getArg(Arg: 0)->IgnoreParenCasts();
7443 StringRef ArgStr = cast<StringLiteral>(Val: ArgExpr)->getString();
7444 llvm::SmallVector<StringRef, 8> OrigFeatures;
7445 ArgStr.split(A&: OrigFeatures, Separator: "+");
7446 llvm::SmallVector<StringRef, 8> Features;
7447 for (StringRef Feature : OrigFeatures) {
7448 Feature = Feature.trim();
7449 if (!llvm::AArch64::parseFMVExtension(Extension: Feature))
7450 return Builder.getFalse();
7451 if (Feature != "default")
7452 Features.push_back(Elt: Feature);
7453 }
7454 return EmitAArch64CpuSupports(FeatureStrs: Features);
7455}
7456
7457llvm::Value *
7458CodeGenFunction::EmitAArch64CpuSupports(ArrayRef<StringRef> FeaturesStrs) {
7459 llvm::APInt FeaturesMask = llvm::AArch64::getCpuSupportsMask(Features: FeaturesStrs);
7460 Value *Result = Builder.getTrue();
7461 if (FeaturesMask != 0) {
7462 // Get features from structure in runtime library
7463 // struct {
7464 // unsigned long long features;
7465 // } __aarch64_cpu_features;
7466 llvm::Type *STy = llvm::StructType::get(elt1: Int64Ty);
7467 llvm::Constant *AArch64CPUFeatures =
7468 CGM.CreateRuntimeVariable(Ty: STy, Name: "__aarch64_cpu_features");
7469 cast<llvm::GlobalValue>(Val: AArch64CPUFeatures)->setDSOLocal(true);
7470 llvm::Value *CpuFeatures = Builder.CreateGEP(
7471 Ty: STy, Ptr: AArch64CPUFeatures,
7472 IdxList: {ConstantInt::get(Ty: Int32Ty, V: 0), ConstantInt::get(Ty: Int32Ty, V: 0)});
7473 Value *Features = Builder.CreateAlignedLoad(Ty: Int64Ty, Addr: CpuFeatures,
7474 Align: CharUnits::fromQuantity(Quantity: 8));
7475 Value *Mask = Builder.getInt(AI: FeaturesMask.trunc(width: 64));
7476 Value *Bitset = Builder.CreateAnd(LHS: Features, RHS: Mask);
7477 Value *Cmp = Builder.CreateICmpEQ(LHS: Bitset, RHS: Mask);
7478 Result = Builder.CreateAnd(LHS: Result, RHS: Cmp);
7479 }
7480 return Result;
7481}
7482