1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/NVVMIntrinsicUtils.h"
47#include "llvm/IR/Value.h"
48#include "llvm/IR/Verifier.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CodeGen.h"
51#include "llvm/Support/CommandLine.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/NVPTXAddrSpace.h"
54#include "llvm/Support/NVVMAttributes.h"
55#include "llvm/Support/Regex.h"
56#include "llvm/Support/TimeProfiler.h"
57#include "llvm/TargetParser/Triple.h"
58#include <cstdint>
59#include <cstring>
60#include <numeric>
61
62using namespace llvm;
63
64static cl::opt<bool>
65 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
66 cl::desc("Disable autoupgrade of debug info"));
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(O&: llvm::errs());
75 llvm::errs() << "\n";
76 reportFatalUsageError(reason);
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
81static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(i: 0);
86 if (Arg0Type != FixedVectorType::get(ElementType: Type::getFloatTy(C&: F->getContext()), NumElts: 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(GV: F);
91 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
97static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 i: F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(BitWidth: 32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(GV: F);
107 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
113static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(GV: F);
120 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
126static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(i: 1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(i: 2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Val: Arg1Type)->getElementType()->isIntegerTy(BitWidth: 8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Val: Arg2Type)->getElementType()->isIntegerTy(BitWidth: 8))
135 return false;
136
137 rename(GV: F);
138 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
144static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(i: 1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(i: 2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Val: Arg1Type)->getElementType()->isIntegerTy(BitWidth: 16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Val: Arg2Type)->getElementType()->isIntegerTy(BitWidth: 16))
153 return false;
154
155 rename(GV: F);
156 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
157 return true;
158}
159
160static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(GV: F);
166 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
167 return true;
168}
169
170static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(i: 1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(GV: F);
176 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
177 return true;
178}
179
180static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front(Prefix: "avx."))
187 return (Name.starts_with(Prefix: "blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with(Prefix: "movnt.") || // Added in 3.2
192 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
193 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
194 Name.starts_with(Prefix: "vbroadcast.s") || // Added in 3.5
195 Name.starts_with(Prefix: "vbroadcastf128") || // Added in 4.0
196 Name.starts_with(Prefix: "vextractf128.") || // Added in 3.7
197 Name.starts_with(Prefix: "vinsertf128.") || // Added in 3.7
198 Name.starts_with(Prefix: "vperm2f128.") || // Added in 6.0
199 Name.starts_with(Prefix: "vpermil.")); // Added in 3.1
200
201 if (Name.consume_front(Prefix: "avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with(Prefix: "pabs.") || // Added in 6.0
204 Name.starts_with(Prefix: "padds.") || // Added in 8.0
205 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
206 Name.starts_with(Prefix: "pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with(Prefix: "pbroadcast") || // Added in 3.8
209 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.1
210 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.1
211 Name.starts_with(Prefix: "pmax") || // Added in 3.9
212 Name.starts_with(Prefix: "pmin") || // Added in 3.9
213 Name.starts_with(Prefix: "pmovsx") || // Added in 3.9
214 Name.starts_with(Prefix: "pmovzx") || // Added in 3.9
215 Name == "pmul.dq" || // Added in 7.0
216 Name == "pmulu.dq" || // Added in 7.0
217 Name.starts_with(Prefix: "psll.dq") || // Added in 3.7
218 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.7
219 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
220 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
221 Name.starts_with(Prefix: "vbroadcast") || // Added in 3.8
222 Name == "vbroadcasti128" || // Added in 3.7
223 Name == "vextracti128" || // Added in 3.7
224 Name == "vinserti128" || // Added in 3.7
225 Name == "vperm2i128"); // Added in 6.0
226
227 if (Name.consume_front(Prefix: "avx512.")) {
228 if (Name.consume_front(Prefix: "mask."))
229 // 'avx512.mask.*'
230 return (Name.starts_with(Prefix: "add.p") || // Added in 7.0. 128/256 in 4.0
231 Name.starts_with(Prefix: "and.") || // Added in 3.9
232 Name.starts_with(Prefix: "andn.") || // Added in 3.9
233 Name.starts_with(Prefix: "broadcast.s") || // Added in 3.9
234 Name.starts_with(Prefix: "broadcastf32x4.") || // Added in 6.0
235 Name.starts_with(Prefix: "broadcastf32x8.") || // Added in 6.0
236 Name.starts_with(Prefix: "broadcastf64x2.") || // Added in 6.0
237 Name.starts_with(Prefix: "broadcastf64x4.") || // Added in 6.0
238 Name.starts_with(Prefix: "broadcasti32x4.") || // Added in 6.0
239 Name.starts_with(Prefix: "broadcasti32x8.") || // Added in 6.0
240 Name.starts_with(Prefix: "broadcasti64x2.") || // Added in 6.0
241 Name.starts_with(Prefix: "broadcasti64x4.") || // Added in 6.0
242 Name.starts_with(Prefix: "cmp.b") || // Added in 5.0
243 Name.starts_with(Prefix: "cmp.d") || // Added in 5.0
244 Name.starts_with(Prefix: "cmp.q") || // Added in 5.0
245 Name.starts_with(Prefix: "cmp.w") || // Added in 5.0
246 Name.starts_with(Prefix: "compress.b") || // Added in 9.0
247 Name.starts_with(Prefix: "compress.d") || // Added in 9.0
248 Name.starts_with(Prefix: "compress.p") || // Added in 9.0
249 Name.starts_with(Prefix: "compress.q") || // Added in 9.0
250 Name.starts_with(Prefix: "compress.store.") || // Added in 7.0
251 Name.starts_with(Prefix: "compress.w") || // Added in 9.0
252 Name.starts_with(Prefix: "conflict.") || // Added in 9.0
253 Name.starts_with(Prefix: "cvtdq2pd.") || // Added in 4.0
254 Name.starts_with(Prefix: "cvtdq2ps.") || // Added in 7.0 updated 9.0
255 Name == "cvtpd2dq.256" || // Added in 7.0
256 Name == "cvtpd2ps.256" || // Added in 7.0
257 Name == "cvtps2pd.128" || // Added in 7.0
258 Name == "cvtps2pd.256" || // Added in 7.0
259 Name.starts_with(Prefix: "cvtqq2pd.") || // Added in 7.0 updated 9.0
260 Name == "cvtqq2ps.256" || // Added in 9.0
261 Name == "cvtqq2ps.512" || // Added in 9.0
262 Name == "cvttpd2dq.256" || // Added in 7.0
263 Name == "cvttps2dq.128" || // Added in 7.0
264 Name == "cvttps2dq.256" || // Added in 7.0
265 Name.starts_with(Prefix: "cvtudq2pd.") || // Added in 4.0
266 Name.starts_with(Prefix: "cvtudq2ps.") || // Added in 7.0 updated 9.0
267 Name.starts_with(Prefix: "cvtuqq2pd.") || // Added in 7.0 updated 9.0
268 Name == "cvtuqq2ps.256" || // Added in 9.0
269 Name == "cvtuqq2ps.512" || // Added in 9.0
270 Name.starts_with(Prefix: "dbpsadbw.") || // Added in 7.0
271 Name.starts_with(Prefix: "div.p") || // Added in 7.0. 128/256 in 4.0
272 Name.starts_with(Prefix: "expand.b") || // Added in 9.0
273 Name.starts_with(Prefix: "expand.d") || // Added in 9.0
274 Name.starts_with(Prefix: "expand.load.") || // Added in 7.0
275 Name.starts_with(Prefix: "expand.p") || // Added in 9.0
276 Name.starts_with(Prefix: "expand.q") || // Added in 9.0
277 Name.starts_with(Prefix: "expand.w") || // Added in 9.0
278 Name.starts_with(Prefix: "fpclass.p") || // Added in 7.0
279 Name.starts_with(Prefix: "insert") || // Added in 4.0
280 Name.starts_with(Prefix: "load.") || // Added in 3.9
281 Name.starts_with(Prefix: "loadu.") || // Added in 3.9
282 Name.starts_with(Prefix: "lzcnt.") || // Added in 5.0
283 Name.starts_with(Prefix: "max.p") || // Added in 7.0. 128/256 in 5.0
284 Name.starts_with(Prefix: "min.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with(Prefix: "movddup") || // Added in 3.9
286 Name.starts_with(Prefix: "move.s") || // Added in 4.0
287 Name.starts_with(Prefix: "movshdup") || // Added in 3.9
288 Name.starts_with(Prefix: "movsldup") || // Added in 3.9
289 Name.starts_with(Prefix: "mul.p") || // Added in 7.0. 128/256 in 4.0
290 Name.starts_with(Prefix: "or.") || // Added in 3.9
291 Name.starts_with(Prefix: "pabs.") || // Added in 6.0
292 Name.starts_with(Prefix: "packssdw.") || // Added in 5.0
293 Name.starts_with(Prefix: "packsswb.") || // Added in 5.0
294 Name.starts_with(Prefix: "packusdw.") || // Added in 5.0
295 Name.starts_with(Prefix: "packuswb.") || // Added in 5.0
296 Name.starts_with(Prefix: "padd.") || // Added in 4.0
297 Name.starts_with(Prefix: "padds.") || // Added in 8.0
298 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
299 Name.starts_with(Prefix: "palignr.") || // Added in 3.9
300 Name.starts_with(Prefix: "pand.") || // Added in 3.9
301 Name.starts_with(Prefix: "pandn.") || // Added in 3.9
302 Name.starts_with(Prefix: "pavg") || // Added in 6.0
303 Name.starts_with(Prefix: "pbroadcast") || // Added in 6.0
304 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.9
305 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.9
306 Name.starts_with(Prefix: "perm.df.") || // Added in 3.9
307 Name.starts_with(Prefix: "perm.di.") || // Added in 3.9
308 Name.starts_with(Prefix: "permvar.") || // Added in 7.0
309 Name.starts_with(Prefix: "pmaddubs.w.") || // Added in 7.0
310 Name.starts_with(Prefix: "pmaddw.d.") || // Added in 7.0
311 Name.starts_with(Prefix: "pmax") || // Added in 4.0
312 Name.starts_with(Prefix: "pmin") || // Added in 4.0
313 Name == "pmov.qd.256" || // Added in 9.0
314 Name == "pmov.qd.512" || // Added in 9.0
315 Name == "pmov.wb.256" || // Added in 9.0
316 Name == "pmov.wb.512" || // Added in 9.0
317 Name.starts_with(Prefix: "pmovsx") || // Added in 4.0
318 Name.starts_with(Prefix: "pmovzx") || // Added in 4.0
319 Name.starts_with(Prefix: "pmul.dq.") || // Added in 4.0
320 Name.starts_with(Prefix: "pmul.hr.sw.") || // Added in 7.0
321 Name.starts_with(Prefix: "pmulh.w.") || // Added in 7.0
322 Name.starts_with(Prefix: "pmulhu.w.") || // Added in 7.0
323 Name.starts_with(Prefix: "pmull.") || // Added in 4.0
324 Name.starts_with(Prefix: "pmultishift.qb.") || // Added in 8.0
325 Name.starts_with(Prefix: "pmulu.dq.") || // Added in 4.0
326 Name.starts_with(Prefix: "por.") || // Added in 3.9
327 Name.starts_with(Prefix: "prol.") || // Added in 8.0
328 Name.starts_with(Prefix: "prolv.") || // Added in 8.0
329 Name.starts_with(Prefix: "pror.") || // Added in 8.0
330 Name.starts_with(Prefix: "prorv.") || // Added in 8.0
331 Name.starts_with(Prefix: "pshuf.b.") || // Added in 4.0
332 Name.starts_with(Prefix: "pshuf.d.") || // Added in 3.9
333 Name.starts_with(Prefix: "pshufh.w.") || // Added in 3.9
334 Name.starts_with(Prefix: "pshufl.w.") || // Added in 3.9
335 Name.starts_with(Prefix: "psll.d") || // Added in 4.0
336 Name.starts_with(Prefix: "psll.q") || // Added in 4.0
337 Name.starts_with(Prefix: "psll.w") || // Added in 4.0
338 Name.starts_with(Prefix: "pslli") || // Added in 4.0
339 Name.starts_with(Prefix: "psllv") || // Added in 4.0
340 Name.starts_with(Prefix: "psra.d") || // Added in 4.0
341 Name.starts_with(Prefix: "psra.q") || // Added in 4.0
342 Name.starts_with(Prefix: "psra.w") || // Added in 4.0
343 Name.starts_with(Prefix: "psrai") || // Added in 4.0
344 Name.starts_with(Prefix: "psrav") || // Added in 4.0
345 Name.starts_with(Prefix: "psrl.d") || // Added in 4.0
346 Name.starts_with(Prefix: "psrl.q") || // Added in 4.0
347 Name.starts_with(Prefix: "psrl.w") || // Added in 4.0
348 Name.starts_with(Prefix: "psrli") || // Added in 4.0
349 Name.starts_with(Prefix: "psrlv") || // Added in 4.0
350 Name.starts_with(Prefix: "psub.") || // Added in 4.0
351 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
352 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
353 Name.starts_with(Prefix: "pternlog.") || // Added in 7.0
354 Name.starts_with(Prefix: "punpckh") || // Added in 3.9
355 Name.starts_with(Prefix: "punpckl") || // Added in 3.9
356 Name.starts_with(Prefix: "pxor.") || // Added in 3.9
357 Name.starts_with(Prefix: "shuf.f") || // Added in 6.0
358 Name.starts_with(Prefix: "shuf.i") || // Added in 6.0
359 Name.starts_with(Prefix: "shuf.p") || // Added in 4.0
360 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
361 Name.starts_with(Prefix: "store.b.") || // Added in 3.9
362 Name.starts_with(Prefix: "store.d.") || // Added in 3.9
363 Name.starts_with(Prefix: "store.p") || // Added in 3.9
364 Name.starts_with(Prefix: "store.q.") || // Added in 3.9
365 Name.starts_with(Prefix: "store.w.") || // Added in 3.9
366 Name == "store.ss" || // Added in 7.0
367 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
368 Name.starts_with(Prefix: "sub.p") || // Added in 7.0. 128/256 in 4.0
369 Name.starts_with(Prefix: "ucmp.") || // Added in 5.0
370 Name.starts_with(Prefix: "unpckh.") || // Added in 3.9
371 Name.starts_with(Prefix: "unpckl.") || // Added in 3.9
372 Name.starts_with(Prefix: "valign.") || // Added in 4.0
373 Name == "vcvtph2ps.128" || // Added in 11.0
374 Name == "vcvtph2ps.256" || // Added in 11.0
375 Name.starts_with(Prefix: "vextract") || // Added in 4.0
376 Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
377 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
378 Name.starts_with(Prefix: "vfnmadd.") || // Added in 7.0
379 Name.starts_with(Prefix: "vfnmsub.") || // Added in 7.0
380 Name.starts_with(Prefix: "vpdpbusd.") || // Added in 7.0
381 Name.starts_with(Prefix: "vpdpbusds.") || // Added in 7.0
382 Name.starts_with(Prefix: "vpdpwssd.") || // Added in 7.0
383 Name.starts_with(Prefix: "vpdpwssds.") || // Added in 7.0
384 Name.starts_with(Prefix: "vpermi2var.") || // Added in 7.0
385 Name.starts_with(Prefix: "vpermil.p") || // Added in 3.9
386 Name.starts_with(Prefix: "vpermilvar.") || // Added in 4.0
387 Name.starts_with(Prefix: "vpermt2var.") || // Added in 7.0
388 Name.starts_with(Prefix: "vpmadd52") || // Added in 7.0
389 Name.starts_with(Prefix: "vpshld.") || // Added in 7.0
390 Name.starts_with(Prefix: "vpshldv.") || // Added in 8.0
391 Name.starts_with(Prefix: "vpshrd.") || // Added in 7.0
392 Name.starts_with(Prefix: "vpshrdv.") || // Added in 8.0
393 Name.starts_with(Prefix: "vpshufbitqmb.") || // Added in 8.0
394 Name.starts_with(Prefix: "xor.")); // Added in 3.9
395
396 if (Name.consume_front(Prefix: "mask3."))
397 // 'avx512.mask3.*'
398 return (Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
399 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
400 Name.starts_with(Prefix: "vfmsub.") || // Added in 7.0
401 Name.starts_with(Prefix: "vfmsubadd.") || // Added in 7.0
402 Name.starts_with(Prefix: "vfnmsub.")); // Added in 7.0
403
404 if (Name.consume_front(Prefix: "maskz."))
405 // 'avx512.maskz.*'
406 return (Name.starts_with(Prefix: "pternlog.") || // Added in 7.0
407 Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
408 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
409 Name.starts_with(Prefix: "vpdpbusd.") || // Added in 7.0
410 Name.starts_with(Prefix: "vpdpbusds.") || // Added in 7.0
411 Name.starts_with(Prefix: "vpdpwssd.") || // Added in 7.0
412 Name.starts_with(Prefix: "vpdpwssds.") || // Added in 7.0
413 Name.starts_with(Prefix: "vpermt2var.") || // Added in 7.0
414 Name.starts_with(Prefix: "vpmadd52") || // Added in 7.0
415 Name.starts_with(Prefix: "vpshldv.") || // Added in 8.0
416 Name.starts_with(Prefix: "vpshrdv.")); // Added in 8.0
417
418 // 'avx512.*'
419 return (Name == "movntdqa" || // Added in 5.0
420 Name == "pmul.dq.512" || // Added in 7.0
421 Name == "pmulu.dq.512" || // Added in 7.0
422 Name.starts_with(Prefix: "broadcastm") || // Added in 6.0
423 Name.starts_with(Prefix: "cmp.p") || // Added in 12.0
424 Name.starts_with(Prefix: "cvtb2mask.") || // Added in 7.0
425 Name.starts_with(Prefix: "cvtd2mask.") || // Added in 7.0
426 Name.starts_with(Prefix: "cvtmask2") || // Added in 5.0
427 Name.starts_with(Prefix: "cvtq2mask.") || // Added in 7.0
428 Name == "cvtusi2sd" || // Added in 7.0
429 Name.starts_with(Prefix: "cvtw2mask.") || // Added in 7.0
430 Name == "kand.w" || // Added in 7.0
431 Name == "kandn.w" || // Added in 7.0
432 Name == "knot.w" || // Added in 7.0
433 Name == "kor.w" || // Added in 7.0
434 Name == "kortestc.w" || // Added in 7.0
435 Name == "kortestz.w" || // Added in 7.0
436 Name.starts_with(Prefix: "kunpck") || // added in 6.0
437 Name == "kxnor.w" || // Added in 7.0
438 Name == "kxor.w" || // Added in 7.0
439 Name.starts_with(Prefix: "padds.") || // Added in 8.0
440 Name.starts_with(Prefix: "pbroadcast") || // Added in 3.9
441 Name.starts_with(Prefix: "prol") || // Added in 8.0
442 Name.starts_with(Prefix: "pror") || // Added in 8.0
443 Name.starts_with(Prefix: "psll.dq") || // Added in 3.9
444 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.9
445 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
446 Name.starts_with(Prefix: "ptestm") || // Added in 6.0
447 Name.starts_with(Prefix: "ptestnm") || // Added in 6.0
448 Name.starts_with(Prefix: "storent.") || // Added in 3.9
449 Name.starts_with(Prefix: "vbroadcast.s") || // Added in 7.0
450 Name.starts_with(Prefix: "vpshld.") || // Added in 8.0
451 Name.starts_with(Prefix: "vpshrd.")); // Added in 8.0
452 }
453
454 if (Name.consume_front(Prefix: "fma."))
455 return (Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
456 Name.starts_with(Prefix: "vfmsub.") || // Added in 7.0
457 Name.starts_with(Prefix: "vfmsubadd.") || // Added in 7.0
458 Name.starts_with(Prefix: "vfnmadd.") || // Added in 7.0
459 Name.starts_with(Prefix: "vfnmsub.")); // Added in 7.0
460
461 if (Name.consume_front(Prefix: "fma4."))
462 return Name.starts_with(Prefix: "vfmadd.s"); // Added in 7.0
463
464 if (Name.consume_front(Prefix: "sse."))
465 return (Name == "add.ss" || // Added in 4.0
466 Name == "cvtsi2ss" || // Added in 7.0
467 Name == "cvtsi642ss" || // Added in 7.0
468 Name == "div.ss" || // Added in 4.0
469 Name == "mul.ss" || // Added in 4.0
470 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
471 Name == "sqrt.ss" || // Added in 7.0
472 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
473 Name == "sub.ss"); // Added in 4.0
474
475 if (Name.consume_front(Prefix: "sse2."))
476 return (Name == "add.sd" || // Added in 4.0
477 Name == "cvtdq2pd" || // Added in 3.9
478 Name == "cvtdq2ps" || // Added in 7.0
479 Name == "cvtps2pd" || // Added in 3.9
480 Name == "cvtsi2sd" || // Added in 7.0
481 Name == "cvtsi642sd" || // Added in 7.0
482 Name == "cvtss2sd" || // Added in 7.0
483 Name == "div.sd" || // Added in 4.0
484 Name == "mul.sd" || // Added in 4.0
485 Name.starts_with(Prefix: "padds.") || // Added in 8.0
486 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
487 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.1
488 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.1
489 Name == "pmaxs.w" || // Added in 3.9
490 Name == "pmaxu.b" || // Added in 3.9
491 Name == "pmins.w" || // Added in 3.9
492 Name == "pminu.b" || // Added in 3.9
493 Name == "pmulu.dq" || // Added in 7.0
494 Name.starts_with(Prefix: "pshuf") || // Added in 3.9
495 Name.starts_with(Prefix: "psll.dq") || // Added in 3.7
496 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.7
497 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
498 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
499 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
500 Name == "sqrt.sd" || // Added in 7.0
501 Name == "storel.dq" || // Added in 3.9
502 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
503 Name == "sub.sd"); // Added in 4.0
504
505 if (Name.consume_front(Prefix: "sse41."))
506 return (Name.starts_with(Prefix: "blendp") || // Added in 3.7
507 Name == "movntdqa" || // Added in 5.0
508 Name == "pblendw" || // Added in 3.7
509 Name == "pmaxsb" || // Added in 3.9
510 Name == "pmaxsd" || // Added in 3.9
511 Name == "pmaxud" || // Added in 3.9
512 Name == "pmaxuw" || // Added in 3.9
513 Name == "pminsb" || // Added in 3.9
514 Name == "pminsd" || // Added in 3.9
515 Name == "pminud" || // Added in 3.9
516 Name == "pminuw" || // Added in 3.9
517 Name.starts_with(Prefix: "pmovsx") || // Added in 3.8
518 Name.starts_with(Prefix: "pmovzx") || // Added in 3.9
519 Name == "pmuldq"); // Added in 7.0
520
521 if (Name.consume_front(Prefix: "sse42."))
522 return Name == "crc32.64.8"; // Added in 3.4
523
524 if (Name.consume_front(Prefix: "sse4a."))
525 return Name.starts_with(Prefix: "movnt."); // Added in 3.9
526
527 if (Name.consume_front(Prefix: "ssse3."))
528 return (Name == "pabs.b.128" || // Added in 6.0
529 Name == "pabs.d.128" || // Added in 6.0
530 Name == "pabs.w.128"); // Added in 6.0
531
532 if (Name.consume_front(Prefix: "xop."))
533 return (Name == "vpcmov" || // Added in 3.8
534 Name == "vpcmov.256" || // Added in 5.0
535 Name.starts_with(Prefix: "vpcom") || // Added in 3.2, Updated in 9.0
536 Name.starts_with(Prefix: "vprot")); // Added in 8.0
537
538 if (Name.consume_front(Prefix: "bmi."))
539 return (Name.starts_with(Prefix: "pdep.") || // Added in 23.0
540 Name.starts_with(Prefix: "pext.")); // Added in 23.0
541
542 return (Name == "addcarry.u32" || // Added in 8.0
543 Name == "addcarry.u64" || // Added in 8.0
544 Name == "addcarryx.u32" || // Added in 8.0
545 Name == "addcarryx.u64" || // Added in 8.0
546 Name == "subborrow.u32" || // Added in 8.0
547 Name == "subborrow.u64" || // Added in 8.0
548 Name.starts_with(Prefix: "vcvtph2ps.")); // Added in 11.0
549}
550
551static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
552 Function *&NewFn) {
553 // Only handle intrinsics that start with "x86.".
554 if (!Name.consume_front(Prefix: "x86."))
555 return false;
556
557 if (shouldUpgradeX86Intrinsic(F, Name)) {
558 NewFn = nullptr;
559 return true;
560 }
561
562 if (Name == "rdtscp") { // Added in 8.0
563 // If this intrinsic has 0 operands, it's the new version.
564 if (F->getFunctionType()->getNumParams() == 0)
565 return false;
566
567 rename(GV: F);
568 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
569 id: Intrinsic::x86_rdtscp);
570 return true;
571 }
572
573 Intrinsic::ID ID;
574
575 // SSE4.1 ptest functions may have an old signature.
576 if (Name.consume_front(Prefix: "sse41.ptest")) { // Added in 3.2
577 ID = StringSwitch<Intrinsic::ID>(Name)
578 .Case(S: "c", Value: Intrinsic::x86_sse41_ptestc)
579 .Case(S: "z", Value: Intrinsic::x86_sse41_ptestz)
580 .Case(S: "nzc", Value: Intrinsic::x86_sse41_ptestnzc)
581 .Default(Value: Intrinsic::not_intrinsic);
582 if (ID != Intrinsic::not_intrinsic)
583 return upgradePTESTIntrinsic(F, IID: ID, NewFn);
584
585 return false;
586 }
587
588 // Several blend and other instructions with masks used the wrong number of
589 // bits.
590
591 // Added in 3.6
592 ID = StringSwitch<Intrinsic::ID>(Name)
593 .Case(S: "sse41.insertps", Value: Intrinsic::x86_sse41_insertps)
594 .Case(S: "sse41.dppd", Value: Intrinsic::x86_sse41_dppd)
595 .Case(S: "sse41.dpps", Value: Intrinsic::x86_sse41_dpps)
596 .Case(S: "sse41.mpsadbw", Value: Intrinsic::x86_sse41_mpsadbw)
597 .Case(S: "avx.dp.ps.256", Value: Intrinsic::x86_avx_dp_ps_256)
598 .Case(S: "avx2.mpsadbw", Value: Intrinsic::x86_avx2_mpsadbw)
599 .Default(Value: Intrinsic::not_intrinsic);
600 if (ID != Intrinsic::not_intrinsic)
601 return upgradeX86IntrinsicsWith8BitMask(F, IID: ID, NewFn);
602
603 if (Name.consume_front(Prefix: "avx512.")) {
604 if (Name.consume_front(Prefix: "mask.cmp.")) {
605 // Added in 7.0
606 ID = StringSwitch<Intrinsic::ID>(Name)
607 .Case(S: "pd.128", Value: Intrinsic::x86_avx512_mask_cmp_pd_128)
608 .Case(S: "pd.256", Value: Intrinsic::x86_avx512_mask_cmp_pd_256)
609 .Case(S: "pd.512", Value: Intrinsic::x86_avx512_mask_cmp_pd_512)
610 .Case(S: "ps.128", Value: Intrinsic::x86_avx512_mask_cmp_ps_128)
611 .Case(S: "ps.256", Value: Intrinsic::x86_avx512_mask_cmp_ps_256)
612 .Case(S: "ps.512", Value: Intrinsic::x86_avx512_mask_cmp_ps_512)
613 .Default(Value: Intrinsic::not_intrinsic);
614 if (ID != Intrinsic::not_intrinsic)
615 return upgradeX86MaskedFPCompare(F, IID: ID, NewFn);
616 } else if (Name.starts_with(Prefix: "vpdpbusd.") ||
617 Name.starts_with(Prefix: "vpdpbusds.")) {
618 // Added in 21.1
619 ID = StringSwitch<Intrinsic::ID>(Name)
620 .Case(S: "vpdpbusd.128", Value: Intrinsic::x86_avx512_vpdpbusd_128)
621 .Case(S: "vpdpbusd.256", Value: Intrinsic::x86_avx512_vpdpbusd_256)
622 .Case(S: "vpdpbusd.512", Value: Intrinsic::x86_avx512_vpdpbusd_512)
623 .Case(S: "vpdpbusds.128", Value: Intrinsic::x86_avx512_vpdpbusds_128)
624 .Case(S: "vpdpbusds.256", Value: Intrinsic::x86_avx512_vpdpbusds_256)
625 .Case(S: "vpdpbusds.512", Value: Intrinsic::x86_avx512_vpdpbusds_512)
626 .Default(Value: Intrinsic::not_intrinsic);
627 if (ID != Intrinsic::not_intrinsic)
628 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
629 } else if (Name.starts_with(Prefix: "vpdpwssd.") ||
630 Name.starts_with(Prefix: "vpdpwssds.")) {
631 // Added in 21.1
632 ID = StringSwitch<Intrinsic::ID>(Name)
633 .Case(S: "vpdpwssd.128", Value: Intrinsic::x86_avx512_vpdpwssd_128)
634 .Case(S: "vpdpwssd.256", Value: Intrinsic::x86_avx512_vpdpwssd_256)
635 .Case(S: "vpdpwssd.512", Value: Intrinsic::x86_avx512_vpdpwssd_512)
636 .Case(S: "vpdpwssds.128", Value: Intrinsic::x86_avx512_vpdpwssds_128)
637 .Case(S: "vpdpwssds.256", Value: Intrinsic::x86_avx512_vpdpwssds_256)
638 .Case(S: "vpdpwssds.512", Value: Intrinsic::x86_avx512_vpdpwssds_512)
639 .Default(Value: Intrinsic::not_intrinsic);
640 if (ID != Intrinsic::not_intrinsic)
641 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
642 }
643 return false; // No other 'x86.avx512.*'.
644 }
645
646 if (Name.consume_front(Prefix: "avx2.")) {
647 if (Name.consume_front(Prefix: "vpdpb")) {
648 // Added in 21.1
649 ID = StringSwitch<Intrinsic::ID>(Name)
650 .Case(S: "ssd.128", Value: Intrinsic::x86_avx2_vpdpbssd_128)
651 .Case(S: "ssd.256", Value: Intrinsic::x86_avx2_vpdpbssd_256)
652 .Case(S: "ssds.128", Value: Intrinsic::x86_avx2_vpdpbssds_128)
653 .Case(S: "ssds.256", Value: Intrinsic::x86_avx2_vpdpbssds_256)
654 .Case(S: "sud.128", Value: Intrinsic::x86_avx2_vpdpbsud_128)
655 .Case(S: "sud.256", Value: Intrinsic::x86_avx2_vpdpbsud_256)
656 .Case(S: "suds.128", Value: Intrinsic::x86_avx2_vpdpbsuds_128)
657 .Case(S: "suds.256", Value: Intrinsic::x86_avx2_vpdpbsuds_256)
658 .Case(S: "uud.128", Value: Intrinsic::x86_avx2_vpdpbuud_128)
659 .Case(S: "uud.256", Value: Intrinsic::x86_avx2_vpdpbuud_256)
660 .Case(S: "uuds.128", Value: Intrinsic::x86_avx2_vpdpbuuds_128)
661 .Case(S: "uuds.256", Value: Intrinsic::x86_avx2_vpdpbuuds_256)
662 .Default(Value: Intrinsic::not_intrinsic);
663 if (ID != Intrinsic::not_intrinsic)
664 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
665 } else if (Name.consume_front(Prefix: "vpdpw")) {
666 // Added in 21.1
667 ID = StringSwitch<Intrinsic::ID>(Name)
668 .Case(S: "sud.128", Value: Intrinsic::x86_avx2_vpdpwsud_128)
669 .Case(S: "sud.256", Value: Intrinsic::x86_avx2_vpdpwsud_256)
670 .Case(S: "suds.128", Value: Intrinsic::x86_avx2_vpdpwsuds_128)
671 .Case(S: "suds.256", Value: Intrinsic::x86_avx2_vpdpwsuds_256)
672 .Case(S: "usd.128", Value: Intrinsic::x86_avx2_vpdpwusd_128)
673 .Case(S: "usd.256", Value: Intrinsic::x86_avx2_vpdpwusd_256)
674 .Case(S: "usds.128", Value: Intrinsic::x86_avx2_vpdpwusds_128)
675 .Case(S: "usds.256", Value: Intrinsic::x86_avx2_vpdpwusds_256)
676 .Case(S: "uud.128", Value: Intrinsic::x86_avx2_vpdpwuud_128)
677 .Case(S: "uud.256", Value: Intrinsic::x86_avx2_vpdpwuud_256)
678 .Case(S: "uuds.128", Value: Intrinsic::x86_avx2_vpdpwuuds_128)
679 .Case(S: "uuds.256", Value: Intrinsic::x86_avx2_vpdpwuuds_256)
680 .Default(Value: Intrinsic::not_intrinsic);
681 if (ID != Intrinsic::not_intrinsic)
682 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
683 }
684 return false; // No other 'x86.avx2.*'
685 }
686
687 if (Name.consume_front(Prefix: "avx10.")) {
688 if (Name.consume_front(Prefix: "vpdpb")) {
689 // Added in 21.1
690 ID = StringSwitch<Intrinsic::ID>(Name)
691 .Case(S: "ssd.512", Value: Intrinsic::x86_avx10_vpdpbssd_512)
692 .Case(S: "ssds.512", Value: Intrinsic::x86_avx10_vpdpbssds_512)
693 .Case(S: "sud.512", Value: Intrinsic::x86_avx10_vpdpbsud_512)
694 .Case(S: "suds.512", Value: Intrinsic::x86_avx10_vpdpbsuds_512)
695 .Case(S: "uud.512", Value: Intrinsic::x86_avx10_vpdpbuud_512)
696 .Case(S: "uuds.512", Value: Intrinsic::x86_avx10_vpdpbuuds_512)
697 .Default(Value: Intrinsic::not_intrinsic);
698 if (ID != Intrinsic::not_intrinsic)
699 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
700 } else if (Name.consume_front(Prefix: "vpdpw")) {
701 ID = StringSwitch<Intrinsic::ID>(Name)
702 .Case(S: "sud.512", Value: Intrinsic::x86_avx10_vpdpwsud_512)
703 .Case(S: "suds.512", Value: Intrinsic::x86_avx10_vpdpwsuds_512)
704 .Case(S: "usd.512", Value: Intrinsic::x86_avx10_vpdpwusd_512)
705 .Case(S: "usds.512", Value: Intrinsic::x86_avx10_vpdpwusds_512)
706 .Case(S: "uud.512", Value: Intrinsic::x86_avx10_vpdpwuud_512)
707 .Case(S: "uuds.512", Value: Intrinsic::x86_avx10_vpdpwuuds_512)
708 .Default(Value: Intrinsic::not_intrinsic);
709 if (ID != Intrinsic::not_intrinsic)
710 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
711 }
712 return false; // No other 'x86.avx10.*'
713 }
714
715 if (Name.consume_front(Prefix: "avx512bf16.")) {
716 // Added in 9.0
717 ID = StringSwitch<Intrinsic::ID>(Name)
718 .Case(S: "cvtne2ps2bf16.128",
719 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
720 .Case(S: "cvtne2ps2bf16.256",
721 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
722 .Case(S: "cvtne2ps2bf16.512",
723 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
724 .Case(S: "mask.cvtneps2bf16.128",
725 Value: Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
726 .Case(S: "cvtneps2bf16.256",
727 Value: Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
728 .Case(S: "cvtneps2bf16.512",
729 Value: Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
730 .Default(Value: Intrinsic::not_intrinsic);
731 if (ID != Intrinsic::not_intrinsic)
732 return upgradeX86BF16Intrinsic(F, IID: ID, NewFn);
733
734 // Added in 9.0
735 ID = StringSwitch<Intrinsic::ID>(Name)
736 .Case(S: "dpbf16ps.128", Value: Intrinsic::x86_avx512bf16_dpbf16ps_128)
737 .Case(S: "dpbf16ps.256", Value: Intrinsic::x86_avx512bf16_dpbf16ps_256)
738 .Case(S: "dpbf16ps.512", Value: Intrinsic::x86_avx512bf16_dpbf16ps_512)
739 .Default(Value: Intrinsic::not_intrinsic);
740 if (ID != Intrinsic::not_intrinsic)
741 return upgradeX86BF16DPIntrinsic(F, IID: ID, NewFn);
742 return false; // No other 'x86.avx512bf16.*'.
743 }
744
745 if (Name.consume_front(Prefix: "xop.")) {
746 Intrinsic::ID ID = Intrinsic::not_intrinsic;
747 if (Name.starts_with(Prefix: "vpermil2")) { // Added in 3.9
748 // Upgrade any XOP PERMIL2 index operand still using a float/double
749 // vector.
750 auto Idx = F->getFunctionType()->getParamType(i: 2);
751 if (Idx->isFPOrFPVectorTy()) {
752 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
753 unsigned EltSize = Idx->getScalarSizeInBits();
754 if (EltSize == 64 && IdxSize == 128)
755 ID = Intrinsic::x86_xop_vpermil2pd;
756 else if (EltSize == 32 && IdxSize == 128)
757 ID = Intrinsic::x86_xop_vpermil2ps;
758 else if (EltSize == 64 && IdxSize == 256)
759 ID = Intrinsic::x86_xop_vpermil2pd_256;
760 else
761 ID = Intrinsic::x86_xop_vpermil2ps_256;
762 }
763 } else if (F->arg_size() == 2)
764 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
765 ID = StringSwitch<Intrinsic::ID>(Name)
766 .Case(S: "vfrcz.ss", Value: Intrinsic::x86_xop_vfrcz_ss)
767 .Case(S: "vfrcz.sd", Value: Intrinsic::x86_xop_vfrcz_sd)
768 .Default(Value: Intrinsic::not_intrinsic);
769
770 if (ID != Intrinsic::not_intrinsic) {
771 rename(GV: F);
772 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
773 return true;
774 }
775 return false; // No other 'x86.xop.*'
776 }
777
778 if (Name == "seh.recoverfp") {
779 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
780 id: Intrinsic::eh_recoverfp);
781 return true;
782 }
783
784 return false;
785}
786
787// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
788// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
789static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
790 StringRef Name,
791 Function *&NewFn) {
792 if (Name.starts_with(Prefix: "rbit")) {
793 // '(arm|aarch64).rbit'.
794 NewFn = Intrinsic::getOrInsertDeclaration(
795 M: F->getParent(), id: Intrinsic::bitreverse, OverloadTys: F->arg_begin()->getType());
796 return true;
797 }
798
799 if (Name == "thread.pointer") {
800 // '(arm|aarch64).thread.pointer'.
801 NewFn = Intrinsic::getOrInsertDeclaration(
802 M: F->getParent(), id: Intrinsic::thread_pointer, OverloadTys: F->getReturnType());
803 return true;
804 }
805
806 bool Neon = Name.consume_front(Prefix: "neon.");
807 if (Neon) {
808 // '(arm|aarch64).neon.*'.
809 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
810 // v16i8 respectively.
811 if (Name.consume_front(Prefix: "bfdot.")) {
812 // (arm|aarch64).neon.bfdot.*'.
813 Intrinsic::ID ID =
814 StringSwitch<Intrinsic::ID>(Name)
815 .Cases(CaseStrings: {"v2f32.v8i8", "v4f32.v16i8"},
816 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
817 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
818 .Default(Value: Intrinsic::not_intrinsic);
819 if (ID != Intrinsic::not_intrinsic) {
820 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
821 assert((OperandWidth == 64 || OperandWidth == 128) &&
822 "Unexpected operand width");
823 LLVMContext &Ctx = F->getParent()->getContext();
824 std::array<Type *, 2> Tys{
825 ._M_elems: {F->getReturnType(),
826 FixedVectorType::get(ElementType: Type::getBFloatTy(C&: Ctx), NumElts: OperandWidth / 16)}};
827 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: Tys);
828 return true;
829 }
830 return false; // No other '(arm|aarch64).neon.bfdot.*'.
831 }
832
833 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
834 // anymore and accept v8bf16 instead of v16i8.
835 if (Name.consume_front(Prefix: "bfm")) {
836 // (arm|aarch64).neon.bfm*'.
837 if (Name.consume_back(Suffix: ".v4f32.v16i8")) {
838 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
839 Intrinsic::ID ID =
840 StringSwitch<Intrinsic::ID>(Name)
841 .Case(S: "mla",
842 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
843 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
844 .Case(S: "lalb",
845 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
846 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
847 .Case(S: "lalt",
848 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
850 .Default(Value: Intrinsic::not_intrinsic);
851 if (ID != Intrinsic::not_intrinsic) {
852 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
853 return true;
854 }
855 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
856 }
857 return false; // No other '(arm|aarch64).neon.bfm*.
858 }
859 // Continue on to Aarch64 Neon or Arm Neon.
860 }
861 // Continue on to Arm or Aarch64.
862
863 if (IsArm) {
864 // 'arm.*'.
865 if (Neon) {
866 // 'arm.neon.*'.
867 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
868 .StartsWith(S: "vclz.", Value: Intrinsic::ctlz)
869 .StartsWith(S: "vcnt.", Value: Intrinsic::ctpop)
870 .StartsWith(S: "vqadds.", Value: Intrinsic::sadd_sat)
871 .StartsWith(S: "vqaddu.", Value: Intrinsic::uadd_sat)
872 .StartsWith(S: "vqsubs.", Value: Intrinsic::ssub_sat)
873 .StartsWith(S: "vqsubu.", Value: Intrinsic::usub_sat)
874 .StartsWith(S: "vrinta.", Value: Intrinsic::round)
875 .StartsWith(S: "vrintn.", Value: Intrinsic::roundeven)
876 .StartsWith(S: "vrintm.", Value: Intrinsic::floor)
877 .StartsWith(S: "vrintp.", Value: Intrinsic::ceil)
878 .StartsWith(S: "vrintx.", Value: Intrinsic::rint)
879 .StartsWith(S: "vrintz.", Value: Intrinsic::trunc)
880 .Default(Value: Intrinsic::not_intrinsic);
881 if (ID != Intrinsic::not_intrinsic) {
882 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
883 OverloadTys: F->arg_begin()->getType());
884 return true;
885 }
886
887 if (Name.consume_front(Prefix: "vst")) {
888 // 'arm.neon.vst*'.
889 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
890 SmallVector<StringRef, 2> Groups;
891 if (vstRegex.match(String: Name, Matches: &Groups)) {
892 static const Intrinsic::ID StoreInts[] = {
893 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
894 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
895
896 static const Intrinsic::ID StoreLaneInts[] = {
897 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
898 Intrinsic::arm_neon_vst4lane};
899
900 auto fArgs = F->getFunctionType()->params();
901 Type *Tys[] = {fArgs[0], fArgs[1]};
902 if (Groups[1].size() == 1)
903 NewFn = Intrinsic::getOrInsertDeclaration(
904 M: F->getParent(), id: StoreInts[fArgs.size() - 3], OverloadTys: Tys);
905 else
906 NewFn = Intrinsic::getOrInsertDeclaration(
907 M: F->getParent(), id: StoreLaneInts[fArgs.size() - 5], OverloadTys: Tys);
908 return true;
909 }
910 return false; // No other 'arm.neon.vst*'.
911 }
912
913 return false; // No other 'arm.neon.*'.
914 }
915
916 if (Name.consume_front(Prefix: "mve.")) {
917 // 'arm.mve.*'.
918 if (Name == "vctp64") {
919 if (cast<FixedVectorType>(Val: F->getReturnType())->getNumElements() == 4) {
920 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
921 // the function and deal with it below in UpgradeIntrinsicCall.
922 rename(GV: F);
923 return true;
924 }
925 return false; // Not 'arm.mve.vctp64'.
926 }
927
928 if (Name.starts_with(Prefix: "vrintn.v")) {
929 NewFn = Intrinsic::getOrInsertDeclaration(
930 M: F->getParent(), id: Intrinsic::roundeven, OverloadTys: F->arg_begin()->getType());
931 return true;
932 }
933
934 // These too are changed to accept a v2i1 instead of the old v4i1.
935 if (Name.consume_back(Suffix: ".v4i1")) {
936 // 'arm.mve.*.v4i1'.
937 if (Name.consume_back(Suffix: ".predicated.v2i64.v4i32"))
938 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
939 return Name == "mull.int" || Name == "vqdmull";
940
941 if (Name.consume_back(Suffix: ".v2i64")) {
942 // 'arm.mve.*.v2i64.v4i1'
943 bool IsGather = Name.consume_front(Prefix: "vldr.gather.");
944 if (IsGather || Name.consume_front(Prefix: "vstr.scatter.")) {
945 if (Name.consume_front(Prefix: "base.")) {
946 // Optional 'wb.' prefix.
947 Name.consume_front(Prefix: "wb.");
948 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
949 // predicated.v2i64.v2i64.v4i1'.
950 return Name == "predicated.v2i64";
951 }
952
953 if (Name.consume_front(Prefix: "offset.predicated."))
954 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
955 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
956
957 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
958 return false;
959 }
960
961 return false; // No other 'arm.mve.*.v2i64.v4i1'.
962 }
963 return false; // No other 'arm.mve.*.v4i1'.
964 }
965 return false; // No other 'arm.mve.*'.
966 }
967
968 if (Name.consume_front(Prefix: "cde.vcx")) {
969 // 'arm.cde.vcx*'.
970 if (Name.consume_back(Suffix: ".predicated.v2i64.v4i1"))
971 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
972 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
973 Name == "3q" || Name == "3qa";
974
975 return false; // No other 'arm.cde.vcx*'.
976 }
977 } else {
978 // 'aarch64.*'.
979 if (Neon) {
980 // 'aarch64.neon.*'.
981 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
982 .StartsWith(S: "frintn", Value: Intrinsic::roundeven)
983 .StartsWith(S: "rbit", Value: Intrinsic::bitreverse)
984 .Default(Value: Intrinsic::not_intrinsic);
985 if (ID != Intrinsic::not_intrinsic) {
986 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
987 OverloadTys: F->arg_begin()->getType());
988 return true;
989 }
990
991 if (Name.starts_with(Prefix: "addp")) {
992 // 'aarch64.neon.addp*'.
993 if (F->arg_size() != 2)
994 return false; // Invalid IR.
995 VectorType *Ty = dyn_cast<VectorType>(Val: F->getReturnType());
996 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
997 NewFn = Intrinsic::getOrInsertDeclaration(
998 M: F->getParent(), id: Intrinsic::aarch64_neon_faddp, OverloadTys: Ty);
999 return true;
1000 }
1001 }
1002
1003 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1004 if (Name.starts_with(Prefix: "bfcvt")) {
1005 NewFn = nullptr;
1006 return true;
1007 }
1008
1009 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1010 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1011 NewFn = nullptr;
1012 return true;
1013 }
1014
1015 return false; // No other 'aarch64.neon.*'.
1016 }
1017 if (Name.consume_front(Prefix: "sve.")) {
1018 // 'aarch64.sve.*'.
1019 if (Name.consume_front(Prefix: "bf")) {
1020 if (Name == "mmla") {
1021 Type *Tys[] = {F->getReturnType(),
1022 std::next(x: F->arg_begin())->getType()};
1023 NewFn = Intrinsic::getOrInsertDeclaration(
1024 M: F->getParent(), id: Intrinsic::aarch64_sve_fmmla, OverloadTys: Tys);
1025 return true;
1026 }
1027 if (Name.consume_back(Suffix: ".lane")) {
1028 // 'aarch64.sve.bf*.lane'.
1029 Intrinsic::ID ID =
1030 StringSwitch<Intrinsic::ID>(Name)
1031 .Case(S: "dot", Value: Intrinsic::aarch64_sve_bfdot_lane_v2)
1032 .Case(S: "mlalb", Value: Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1033 .Case(S: "mlalt", Value: Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1034 .Default(Value: Intrinsic::not_intrinsic);
1035 if (ID != Intrinsic::not_intrinsic) {
1036 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1037 return true;
1038 }
1039 return false; // No other 'aarch64.sve.bf*.lane'.
1040 }
1041 return false; // No other 'aarch64.sve.bf*'.
1042 }
1043
1044 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1045 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1046 NewFn = nullptr;
1047 return true;
1048 }
1049
1050 if (Name.consume_front(Prefix: "convert.from.svbool")) {
1051 // 'aarch64.sve.convert.from.svbool'
1052 auto *TTy = dyn_cast<TargetExtType>(Val: F->getReturnType());
1053 if (!TTy || TTy->getName() != "aarch64.svcount")
1054 return false;
1055
1056 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1057 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1058 return true;
1059 }
1060
1061 if (Name.consume_front(Prefix: "convert.to.svbool")) {
1062 // 'aarch64.sve.convert.to.svbool'
1063 auto *TTy = dyn_cast<TargetExtType>(Val: F->arg_begin()->getType());
1064 if (!TTy || TTy->getName() != "aarch64.svcount")
1065 return false;
1066
1067 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1068 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1069 return true;
1070 }
1071
1072 if (Name.consume_front(Prefix: "addqv")) {
1073 // 'aarch64.sve.addqv'.
1074 if (!F->getReturnType()->isFPOrFPVectorTy())
1075 return false;
1076
1077 auto Args = F->getFunctionType()->params();
1078 Type *Tys[] = {F->getReturnType(), Args[1]};
1079 NewFn = Intrinsic::getOrInsertDeclaration(
1080 M: F->getParent(), id: Intrinsic::aarch64_sve_faddqv, OverloadTys: Tys);
1081 return true;
1082 }
1083
1084 if (Name.consume_front(Prefix: "ld")) {
1085 // 'aarch64.sve.ld*'.
1086 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1087 if (LdRegex.match(String: Name)) {
1088 Type *ScalarTy =
1089 cast<VectorType>(Val: F->getReturnType())->getElementType();
1090 ElementCount EC =
1091 cast<VectorType>(Val: F->arg_begin()->getType())->getElementCount();
1092 assert(F->arg_size() == 2 &&
1093 "Expected 2 arguments for ld* intrinsic.");
1094 Type *PtrTy = F->getArg(i: 1)->getType();
1095 Type *Ty = VectorType::get(ElementType: ScalarTy, EC);
1096 static const Intrinsic::ID LoadIDs[] = {
1097 Intrinsic::aarch64_sve_ld2_sret,
1098 Intrinsic::aarch64_sve_ld3_sret,
1099 Intrinsic::aarch64_sve_ld4_sret,
1100 };
1101 NewFn = Intrinsic::getOrInsertDeclaration(
1102 M: F->getParent(), id: LoadIDs[Name[0] - '2'], OverloadTys: {Ty, PtrTy});
1103 return true;
1104 }
1105 return false; // No other 'aarch64.sve.ld*'.
1106 }
1107
1108 if (Name.consume_front(Prefix: "tuple.")) {
1109 // 'aarch64.sve.tuple.*'.
1110 if (Name.starts_with(Prefix: "get")) {
1111 // 'aarch64.sve.tuple.get*'.
1112 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1113 NewFn = Intrinsic::getOrInsertDeclaration(
1114 M: F->getParent(), id: Intrinsic::vector_extract, OverloadTys: Tys);
1115 return true;
1116 }
1117
1118 if (Name.starts_with(Prefix: "set")) {
1119 // 'aarch64.sve.tuple.set*'.
1120 auto Args = F->getFunctionType()->params();
1121 Type *Tys[] = {Args[0], Args[2], Args[1]};
1122 NewFn = Intrinsic::getOrInsertDeclaration(
1123 M: F->getParent(), id: Intrinsic::vector_insert, OverloadTys: Tys);
1124 return true;
1125 }
1126
1127 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1128 if (CreateTupleRegex.match(String: Name)) {
1129 // 'aarch64.sve.tuple.create*'.
1130 auto Args = F->getFunctionType()->params();
1131 Type *Tys[] = {F->getReturnType(), Args[1]};
1132 NewFn = Intrinsic::getOrInsertDeclaration(
1133 M: F->getParent(), id: Intrinsic::vector_insert, OverloadTys: Tys);
1134 return true;
1135 }
1136 return false; // No other 'aarch64.sve.tuple.*'.
1137 }
1138
1139 if (Name.starts_with(Prefix: "rev.nxv")) {
1140 // 'aarch64.sve.rev.<Ty>'
1141 NewFn = Intrinsic::getOrInsertDeclaration(
1142 M: F->getParent(), id: Intrinsic::vector_reverse, OverloadTys: F->getReturnType());
1143 return true;
1144 }
1145
1146 return false; // No other 'aarch64.sve.*'.
1147 }
1148 if (Name.consume_front(Prefix: "sme.")) {
1149 // 'aarch64.sme.*'.
1150 if (Name.consume_front(Prefix: "ftmopa.")) {
1151 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1152 // intrinsics to model their FPMR dependency.
1153 Intrinsic::ID ID =
1154 StringSwitch<Intrinsic::ID>(Name)
1155 .Case(S: "za16.nxv16i8", Value: Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1156 .Case(S: "za32.nxv16i8", Value: Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1157 .Default(Value: Intrinsic::not_intrinsic);
1158 if (ID != Intrinsic::not_intrinsic) {
1159 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1160 return true;
1161 }
1162 return false; // No other 'aarch64.sme.ftmopa.*'.
1163 }
1164
1165 return false; // No other 'aarch64.sme.*'.
1166 }
1167 }
1168 return false; // No other 'arm.*', 'aarch64.*'.
1169}
1170
1171static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F,
1172 StringRef Name) {
1173 if (Name.consume_front(Prefix: "cp.async.bulk.tensor.g2s.")) {
1174 Intrinsic::ID ID =
1175 StringSwitch<Intrinsic::ID>(Name)
1176 .Case(S: "im2col.3d",
1177 Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)
1178 .Case(S: "im2col.4d",
1179 Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)
1180 .Case(S: "im2col.5d",
1181 Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)
1182 .Case(S: "tile.1d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)
1183 .Case(S: "tile.2d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)
1184 .Case(S: "tile.3d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)
1185 .Case(S: "tile.4d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)
1186 .Case(S: "tile.5d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)
1187 .Default(Value: Intrinsic::not_intrinsic);
1188
1189 if (ID == Intrinsic::not_intrinsic)
1190 return ID;
1191
1192 // These intrinsics may need upgrade for two reasons:
1193 // (1) When the address-space of the first argument is shared[AS=3]
1194 // (and we upgrade it to use shared_cluster address-space[AS=7])
1195 if (F->getArg(i: 0)->getType()->getPointerAddressSpace() ==
1196 NVPTXAS::ADDRESS_SPACE_SHARED)
1197 return ID;
1198
1199 // (2) When there are only two boolean flag arguments at the end:
1200 //
1201 // The last three parameters of the older version of these
1202 // intrinsics are: arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag
1203 //
1204 // The newer version reads as:
1205 // arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag, i32 cta_group_flag
1206 //
1207 // So, when the type of the [N-3]rd argument is "not i1", then
1208 // it is the older version and we need to upgrade.
1209 size_t FlagStartIndex = F->getFunctionType()->getNumParams() - 3;
1210 Type *ArgType = F->getFunctionType()->getParamType(i: FlagStartIndex);
1211 if (!ArgType->isIntegerTy(BitWidth: 1))
1212 return ID;
1213 }
1214
1215 return Intrinsic::not_intrinsic;
1216}
1217
1218// The legacy TMA reduction intrinsics encode the reduction operator in their
1219// name, while the current ones take it as an immediate argument. Map the
1220// operator part of a legacy name to the corresponding immediate value.
1221static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1222 return StringSwitch<std::optional<unsigned>>(Name)
1223 .Case(S: "add", Value: static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1224 .Case(S: "min", Value: static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1225 .Case(S: "max", Value: static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1226 .Case(S: "inc", Value: static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1227 .Case(S: "dec", Value: static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1228 .Case(S: "and", Value: static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1229 .Case(S: "or", Value: static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1230 .Case(S: "xor", Value: static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1231 .Default(Value: std::nullopt);
1232}
1233
1234static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1235 if (!Name.consume_front(Prefix: "cp.async.bulk.tensor.reduce."))
1236 return Intrinsic::not_intrinsic;
1237
1238 auto [RedOpName, ShapeName] = Name.split(Separator: '.');
1239 if (!getNVPTXTMAReductionOp(Name: RedOpName))
1240 return Intrinsic::not_intrinsic;
1241
1242 return StringSwitch<Intrinsic::ID>(ShapeName)
1243 .Case(S: "tile.1d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1244 .Case(S: "tile.2d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1245 .Case(S: "tile.3d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1246 .Case(S: "tile.4d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1247 .Case(S: "tile.5d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1248 .Case(S: "im2col.3d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1249 .Case(S: "im2col.4d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1250 .Case(S: "im2col.5d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1251 .Default(Value: Intrinsic::not_intrinsic);
1252}
1253
1254static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1255 StringRef Name) {
1256 if (Name.consume_front(Prefix: "mapa.shared.cluster"))
1257 if (F->getReturnType()->getPointerAddressSpace() ==
1258 NVPTXAS::ADDRESS_SPACE_SHARED)
1259 return Intrinsic::nvvm_mapa_shared_cluster;
1260
1261 if (Name.consume_front(Prefix: "cp.async.bulk.")) {
1262 Intrinsic::ID ID =
1263 StringSwitch<Intrinsic::ID>(Name)
1264 .Case(S: "global.to.shared.cluster",
1265 Value: Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
1266 .Case(S: "shared.cta.to.cluster",
1267 Value: Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1268 .Default(Value: Intrinsic::not_intrinsic);
1269
1270 if (ID != Intrinsic::not_intrinsic)
1271 if (F->getArg(i: 0)->getType()->getPointerAddressSpace() ==
1272 NVPTXAS::ADDRESS_SPACE_SHARED)
1273 return ID;
1274 }
1275
1276 return Intrinsic::not_intrinsic;
1277}
1278
1279static Intrinsic::ID
1280shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1281 if (!Name.consume_front(Prefix: "tcgen05.commit."))
1282 return Intrinsic::not_intrinsic;
1283
1284 if (Name.consume_front(Prefix: "shared."))
1285 return StringSwitch<Intrinsic::ID>(Name)
1286 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_commit_cg1)
1287 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_commit_cg2)
1288 .Default(Value: Intrinsic::not_intrinsic);
1289
1290 if (Name.consume_front(Prefix: "mc.shared.")) {
1291 // Only upgrade older i16 mc variants.
1292 if (!F->getArg(i: 1)->getType()->isIntegerTy(BitWidth: 16))
1293 return Intrinsic::not_intrinsic;
1294
1295 return StringSwitch<Intrinsic::ID>(Name)
1296 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1297 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1298 .Default(Value: Intrinsic::not_intrinsic);
1299 }
1300
1301 return Intrinsic::not_intrinsic;
1302}
1303
1304static Intrinsic::ID
1305shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1306 if (F->arg_size() != 2)
1307 return Intrinsic::not_intrinsic;
1308
1309 if (Name.consume_front(Prefix: "tcgen05.alloc.shared.") ||
1310 Name.consume_front(Prefix: "tcgen05.alloc."))
1311 return StringSwitch<Intrinsic::ID>(Name)
1312 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_alloc_cg1)
1313 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_alloc_cg2)
1314 .Default(Value: Intrinsic::not_intrinsic);
1315
1316 if (Name.consume_front(Prefix: "tcgen05.dealloc."))
1317 return StringSwitch<Intrinsic::ID>(Name)
1318 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_dealloc_cg1)
1319 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_dealloc_cg2)
1320 .Default(Value: Intrinsic::not_intrinsic);
1321
1322 return Intrinsic::not_intrinsic;
1323}
1324
1325static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1326 if (Name.consume_front(Prefix: "fma.rn."))
1327 return StringSwitch<Intrinsic::ID>(Name)
1328 .Case(S: "bf16", Value: Intrinsic::nvvm_fma_rn_bf16)
1329 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fma_rn_bf16x2)
1330 .Case(S: "relu.bf16", Value: Intrinsic::nvvm_fma_rn_relu_bf16)
1331 .Case(S: "relu.bf16x2", Value: Intrinsic::nvvm_fma_rn_relu_bf16x2)
1332 .Default(Value: Intrinsic::not_intrinsic);
1333
1334 if (Name.consume_front(Prefix: "fmax."))
1335 return StringSwitch<Intrinsic::ID>(Name)
1336 .Case(S: "bf16", Value: Intrinsic::nvvm_fmax_bf16)
1337 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fmax_bf16x2)
1338 .Case(S: "ftz.bf16", Value: Intrinsic::nvvm_fmax_ftz_bf16)
1339 .Case(S: "ftz.bf16x2", Value: Intrinsic::nvvm_fmax_ftz_bf16x2)
1340 .Case(S: "ftz.nan.bf16", Value: Intrinsic::nvvm_fmax_ftz_nan_bf16)
1341 .Case(S: "ftz.nan.bf16x2", Value: Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1342 .Case(S: "ftz.nan.xorsign.abs.bf16",
1343 Value: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1344 .Case(S: "ftz.nan.xorsign.abs.bf16x2",
1345 Value: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1346 .Case(S: "ftz.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1347 .Case(S: "ftz.xorsign.abs.bf16x2",
1348 Value: Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1349 .Case(S: "nan.bf16", Value: Intrinsic::nvvm_fmax_nan_bf16)
1350 .Case(S: "nan.bf16x2", Value: Intrinsic::nvvm_fmax_nan_bf16x2)
1351 .Case(S: "nan.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1352 .Case(S: "nan.xorsign.abs.bf16x2",
1353 Value: Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1354 .Case(S: "xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1355 .Case(S: "xorsign.abs.bf16x2", Value: Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1356 .Default(Value: Intrinsic::not_intrinsic);
1357
1358 if (Name.consume_front(Prefix: "fmin."))
1359 return StringSwitch<Intrinsic::ID>(Name)
1360 .Case(S: "bf16", Value: Intrinsic::nvvm_fmin_bf16)
1361 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fmin_bf16x2)
1362 .Case(S: "ftz.bf16", Value: Intrinsic::nvvm_fmin_ftz_bf16)
1363 .Case(S: "ftz.bf16x2", Value: Intrinsic::nvvm_fmin_ftz_bf16x2)
1364 .Case(S: "ftz.nan.bf16", Value: Intrinsic::nvvm_fmin_ftz_nan_bf16)
1365 .Case(S: "ftz.nan.bf16x2", Value: Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1366 .Case(S: "ftz.nan.xorsign.abs.bf16",
1367 Value: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1368 .Case(S: "ftz.nan.xorsign.abs.bf16x2",
1369 Value: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1370 .Case(S: "ftz.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1371 .Case(S: "ftz.xorsign.abs.bf16x2",
1372 Value: Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1373 .Case(S: "nan.bf16", Value: Intrinsic::nvvm_fmin_nan_bf16)
1374 .Case(S: "nan.bf16x2", Value: Intrinsic::nvvm_fmin_nan_bf16x2)
1375 .Case(S: "nan.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1376 .Case(S: "nan.xorsign.abs.bf16x2",
1377 Value: Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1378 .Case(S: "xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1379 .Case(S: "xorsign.abs.bf16x2", Value: Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1380 .Default(Value: Intrinsic::not_intrinsic);
1381
1382 if (Name.consume_front(Prefix: "neg."))
1383 return StringSwitch<Intrinsic::ID>(Name)
1384 .Case(S: "bf16", Value: Intrinsic::nvvm_neg_bf16)
1385 .Case(S: "bf16x2", Value: Intrinsic::nvvm_neg_bf16x2)
1386 .Default(Value: Intrinsic::not_intrinsic);
1387
1388 return Intrinsic::not_intrinsic;
1389}
1390
1391static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
1392 StringRef Name) {
1393 if (!Name.consume_front(Prefix: "tcgen05.mma."))
1394 return Intrinsic::not_intrinsic;
1395
1396 // tcgen05.mma.ws.* variants do not need collector-b appended.
1397 if (Name.starts_with(Prefix: "ws"))
1398 return Intrinsic::not_intrinsic;
1399
1400 return F->getIntrinsicID();
1401}
1402
1403static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1404 return Name.consume_front(Prefix: "local") || Name.consume_front(Prefix: "shared") ||
1405 Name.consume_front(Prefix: "global") || Name.consume_front(Prefix: "constant") ||
1406 Name.consume_front(Prefix: "param");
1407}
1408
1409static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1410 if (!Name.consume_front(Prefix: "vp."))
1411 return 0;
1412 return StringSwitch<unsigned>(Name)
1413 .StartsWith(S: "select", Value: Instruction::Select)
1414 .StartsWith(S: "add", Value: Instruction::Add)
1415 .StartsWith(S: "sub", Value: Instruction::Sub)
1416 .StartsWith(S: "mul", Value: Instruction::Mul)
1417 .StartsWith(S: "ashr", Value: Instruction::AShr)
1418 .StartsWith(S: "lshr", Value: Instruction::LShr)
1419 .StartsWith(S: "shl", Value: Instruction::Shl)
1420 .StartsWith(S: "or", Value: Instruction::Or)
1421 .StartsWith(S: "and", Value: Instruction::And)
1422 .StartsWith(S: "xor", Value: Instruction::Xor)
1423 .StartsWith(S: "fadd", Value: Instruction::FAdd)
1424 .StartsWith(S: "fsub", Value: Instruction::FSub)
1425 .StartsWith(S: "fmuladd", Value: 0)
1426 .StartsWith(S: "fmul", Value: Instruction::FMul)
1427 .StartsWith(S: "fdiv", Value: Instruction::FDiv)
1428 .StartsWith(S: "frem", Value: Instruction::FRem)
1429 .StartsWith(S: "fneg", Value: Instruction::FNeg)
1430 .StartsWith(S: "trunc", Value: Instruction::Trunc)
1431 .StartsWith(S: "zext", Value: Instruction::ZExt)
1432 .StartsWith(S: "sext", Value: Instruction::SExt)
1433 .StartsWith(S: "fptrunc", Value: Instruction::FPTrunc)
1434 .StartsWith(S: "fpext", Value: Instruction::FPExt)
1435 .StartsWith(S: "fptoui", Value: Instruction::FPToUI)
1436 .StartsWith(S: "fptosi", Value: Instruction::FPToSI)
1437 .StartsWith(S: "uitofp", Value: Instruction::UIToFP)
1438 .StartsWith(S: "sitofp", Value: Instruction::SIToFP)
1439 .StartsWith(S: "ptrtoint", Value: Instruction::PtrToInt)
1440 .StartsWith(S: "inttoptr", Value: Instruction::IntToPtr)
1441 .StartsWith(S: "icmp", Value: Instruction::ICmp)
1442 .StartsWith(S: "fcmp", Value: Instruction::FCmp)
1443 .Default(Value: 0);
1444}
1445
1446static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1447 if (!Name.consume_front(Prefix: "vp."))
1448 return 0;
1449 return StringSwitch<Intrinsic::ID>(Name)
1450 .StartsWith(S: "abs", Value: Intrinsic::abs)
1451 .StartsWith(S: "smax", Value: Intrinsic::smax)
1452 .StartsWith(S: "smin", Value: Intrinsic::smin)
1453 .StartsWith(S: "umax", Value: Intrinsic::umax)
1454 .StartsWith(S: "umin", Value: Intrinsic::umin)
1455 .StartsWith(S: "copysign", Value: Intrinsic::copysign)
1456 .StartsWith(S: "minnum", Value: Intrinsic::minnum)
1457 .StartsWith(S: "maxnum", Value: Intrinsic::maxnum)
1458 .StartsWith(S: "minimum", Value: Intrinsic::minimum)
1459 .StartsWith(S: "maximum", Value: Intrinsic::maximum)
1460 .StartsWith(S: "fabs", Value: Intrinsic::fabs)
1461 .StartsWith(S: "sqrt", Value: Intrinsic::sqrt)
1462 .StartsWith(S: "fma", Value: Intrinsic::fma)
1463 .StartsWith(S: "fmuladd", Value: Intrinsic::fmuladd)
1464 .StartsWith(S: "ceil", Value: Intrinsic::ceil)
1465 .StartsWith(S: "floor", Value: Intrinsic::floor)
1466 .StartsWith(S: "rint", Value: Intrinsic::rint)
1467 .StartsWith(S: "nearbyint", Value: Intrinsic::nearbyint)
1468 .StartsWith(S: "roundeven", Value: Intrinsic::roundeven)
1469 .StartsWith(S: "roundtozero", Value: Intrinsic::trunc)
1470 .StartsWith(S: "round", Value: Intrinsic::round)
1471 .StartsWith(S: "lrint", Value: Intrinsic::lrint)
1472 .StartsWith(S: "llrint", Value: Intrinsic::llrint)
1473 .StartsWith(S: "bitreverse", Value: Intrinsic::bitreverse)
1474 .StartsWith(S: "bswap", Value: Intrinsic::bswap)
1475 .StartsWith(S: "ctpop", Value: Intrinsic::ctpop)
1476 .StartsWith(S: "ctlz", Value: Intrinsic::ctlz)
1477 .StartsWith(S: "cttz.elts", Value: 0)
1478 .StartsWith(S: "cttz", Value: Intrinsic::cttz)
1479 .StartsWith(S: "sadd.sat", Value: Intrinsic::sadd_sat)
1480 .StartsWith(S: "uadd.sat", Value: Intrinsic::uadd_sat)
1481 .StartsWith(S: "ssub.sat", Value: Intrinsic::ssub_sat)
1482 .StartsWith(S: "usub.sat", Value: Intrinsic::usub_sat)
1483 .StartsWith(S: "fshl", Value: Intrinsic::fshl)
1484 .StartsWith(S: "fshr", Value: Intrinsic::fshr)
1485 .StartsWith(S: "is.fpclass", Value: Intrinsic::is_fpclass)
1486 .Default(Value: 0);
1487}
1488
1489static bool shouldUpgradeVPIntrinsic(StringRef Name) {
1490 return getFunctionalOpcodeForVP(Name) || getFunctionalIntrinsicIDForVP(Name);
1491}
1492
1493static bool convertIntrinsicValidType(StringRef Name,
1494 const FunctionType *FuncTy) {
1495 Type *HalfTy = Type::getHalfTy(C&: FuncTy->getContext());
1496 if (Name.starts_with(Prefix: "to.fp16")) {
1497 return CastInst::castIsValid(op: Instruction::FPTrunc, SrcTy: FuncTy->getParamType(i: 0),
1498 DstTy: HalfTy) &&
1499 CastInst::castIsValid(op: Instruction::BitCast, SrcTy: HalfTy,
1500 DstTy: FuncTy->getReturnType());
1501 }
1502
1503 if (Name.starts_with(Prefix: "from.fp16")) {
1504 return CastInst::castIsValid(op: Instruction::BitCast, SrcTy: FuncTy->getParamType(i: 0),
1505 DstTy: HalfTy) &&
1506 CastInst::castIsValid(op: Instruction::FPExt, SrcTy: HalfTy,
1507 DstTy: FuncTy->getReturnType());
1508 }
1509
1510 return false;
1511}
1512
1513static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn) {
1514 Intrinsic::ID IID = Intrinsic::lookupIntrinsicID(Name: F->getName());
1515 if (IID == Intrinsic::not_intrinsic)
1516 return false;
1517
1518 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1519 if (Defaults.empty())
1520 return false;
1521
1522 // Overloaded intrinsics are out of scope for the default-arg feature
1523 // and will be supported in a follow-up.
1524 if (Intrinsic::isOverloaded(id: IID))
1525 return false;
1526
1527 // Get the canonical full declaration for this intrinsic.
1528 Function *FullDecl = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
1529
1530 // If the existing declaration already has all args, nothing to upgrade
1531 if (F->arg_size() >= FullDecl->arg_size())
1532 return false;
1533
1534 // Defaults are a contiguous trailing block, so checking the first missing
1535 // argument is enough.
1536 if (F->arg_size() < FirstDefault)
1537 return false;
1538
1539 NewFn = FullDecl;
1540 return true;
1541}
1542
1543static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1544 bool CanUpgradeDebugIntrinsicsToRecords) {
1545 assert(F && "Illegal to upgrade a non-existent Function.");
1546
1547 StringRef Name = F->getName();
1548
1549 // Quickly eliminate it, if it's not a candidate.
1550 if (!Name.consume_front(Prefix: "llvm.") || Name.empty())
1551 return false;
1552
1553 switch (Name[0]) {
1554 default: break;
1555 case 'a': {
1556 bool IsArm = Name.consume_front(Prefix: "arm.");
1557 if (IsArm || Name.consume_front(Prefix: "aarch64.")) {
1558 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1559 return true;
1560 break;
1561 }
1562
1563 if (Name.consume_front(Prefix: "amdgcn.")) {
1564 if (Name == "alignbit") {
1565 // Target specific intrinsic became redundant
1566 NewFn = Intrinsic::getOrInsertDeclaration(
1567 M: F->getParent(), id: Intrinsic::fshr, OverloadTys: {F->getReturnType()});
1568 return true;
1569 }
1570
1571 if (Name.consume_front(Prefix: "atomic.")) {
1572 if (Name.starts_with(Prefix: "inc") || Name.starts_with(Prefix: "dec") ||
1573 Name.starts_with(Prefix: "cond.sub") || Name.starts_with(Prefix: "csub")) {
1574 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1575 // and usub_sat so there's no new declaration.
1576 NewFn = nullptr;
1577 return true;
1578 }
1579 break; // No other 'amdgcn.atomic.*'
1580 }
1581
1582 switch (F->getIntrinsicID()) {
1583 default:
1584 break;
1585 // Legacy wmma iu intrinsics without the optional clamp operand.
1586 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1587 if (F->arg_size() == 7) {
1588 NewFn = nullptr;
1589 return true;
1590 }
1591 break;
1592 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1593 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1594 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1595 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1596 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1597 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1598 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1599 if (F->arg_size() == 8) {
1600 NewFn = nullptr;
1601 return true;
1602 }
1603 break;
1604 }
1605
1606 if (Name.consume_front(Prefix: "ds.") || Name.consume_front(Prefix: "global.atomic.") ||
1607 Name.consume_front(Prefix: "flat.atomic.")) {
1608 if (Name.starts_with(Prefix: "fadd") ||
1609 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1610 (Name.starts_with(Prefix: "fmin") && !Name.starts_with(Prefix: "fmin.num")) ||
1611 (Name.starts_with(Prefix: "fmax") && !Name.starts_with(Prefix: "fmax.num"))) {
1612 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1613 // declaration.
1614 NewFn = nullptr;
1615 return true;
1616 }
1617 }
1618
1619 if (Name.starts_with(Prefix: "fcmp.") || Name.starts_with(Prefix: "icmp.")) {
1620 NewFn = nullptr;
1621 return true;
1622 }
1623
1624 if (Name.starts_with(Prefix: "ldexp.")) {
1625 // Target specific intrinsic became redundant
1626 NewFn = Intrinsic::getOrInsertDeclaration(
1627 M: F->getParent(), id: Intrinsic::ldexp,
1628 OverloadTys: {F->getReturnType(), F->getArg(i: 1)->getType()});
1629 return true;
1630 }
1631 break; // No other 'amdgcn.*'
1632 }
1633
1634 break;
1635 }
1636 case 'c': {
1637 if (F->arg_size() == 1) {
1638 if (Name.consume_front(Prefix: "convert.")) {
1639 if (convertIntrinsicValidType(Name, FuncTy: F->getFunctionType())) {
1640 NewFn = nullptr;
1641 return true;
1642 }
1643 }
1644
1645 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1646 .StartsWith(S: "ctlz.", Value: Intrinsic::ctlz)
1647 .StartsWith(S: "cttz.", Value: Intrinsic::cttz)
1648 .Default(Value: Intrinsic::not_intrinsic);
1649 if (ID != Intrinsic::not_intrinsic) {
1650 rename(GV: F);
1651 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1652 OverloadTys: F->arg_begin()->getType());
1653 return true;
1654 }
1655 }
1656
1657 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1658 if (Name == "coro.end" &&
1659 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(BitWidth: 1)))
1660 CoroEndID = Intrinsic::coro_end;
1661 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(BitWidth: 1))
1662 CoroEndID = Intrinsic::coro_end_async;
1663
1664 if (CoroEndID != Intrinsic::not_intrinsic) {
1665 rename(GV: F);
1666 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: CoroEndID);
1667 return true;
1668 }
1669
1670 break;
1671 }
1672 case 'd':
1673 if (Name.consume_front(Prefix: "dbg.")) {
1674 // Mark debug intrinsics for upgrade to new debug format.
1675 if (CanUpgradeDebugIntrinsicsToRecords) {
1676 if (Name == "addr" || Name == "value" || Name == "assign" ||
1677 Name == "declare" || Name == "label") {
1678 // There's no function to replace these with.
1679 NewFn = nullptr;
1680 // But we do want these to get upgraded.
1681 return true;
1682 }
1683 }
1684 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1685 // converted to DbgVariableRecords later.
1686 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1687 rename(GV: F);
1688 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1689 id: Intrinsic::dbg_value);
1690 return true;
1691 }
1692 break; // No other 'dbg.*'.
1693 }
1694 break;
1695 case 'e':
1696 if (Name.consume_front(Prefix: "experimental.vector.")) {
1697 Intrinsic::ID ID =
1698 StringSwitch<Intrinsic::ID>(Name)
1699 // Skip over extract.last.active, otherwise it will be 'upgraded'
1700 // to a regular vector extract which is a different operation.
1701 .StartsWith(S: "extract.last.active.", Value: Intrinsic::not_intrinsic)
1702 .StartsWith(S: "extract.", Value: Intrinsic::vector_extract)
1703 .StartsWith(S: "insert.", Value: Intrinsic::vector_insert)
1704 .StartsWith(S: "reverse.", Value: Intrinsic::vector_reverse)
1705 .StartsWith(S: "interleave2.", Value: Intrinsic::vector_interleave2)
1706 .StartsWith(S: "deinterleave2.", Value: Intrinsic::vector_deinterleave2)
1707 .StartsWith(S: "partial.reduce.add",
1708 Value: Intrinsic::vector_partial_reduce_add)
1709 .Default(Value: Intrinsic::not_intrinsic);
1710 if (ID != Intrinsic::not_intrinsic) {
1711 const auto *FT = F->getFunctionType();
1712 SmallVector<Type *, 2> Tys;
1713 if (ID == Intrinsic::vector_extract ||
1714 ID == Intrinsic::vector_interleave2)
1715 // Extracting overloads the return type.
1716 Tys.push_back(Elt: FT->getReturnType());
1717 if (ID != Intrinsic::vector_interleave2)
1718 Tys.push_back(Elt: FT->getParamType(i: 0));
1719 if (ID == Intrinsic::vector_insert ||
1720 ID == Intrinsic::vector_partial_reduce_add)
1721 // Inserting overloads the inserted type.
1722 Tys.push_back(Elt: FT->getParamType(i: 1));
1723 rename(GV: F);
1724 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: Tys);
1725 return true;
1726 }
1727
1728 if (Name.consume_front(Prefix: "reduce.")) {
1729 SmallVector<StringRef, 2> Groups;
1730 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1731 if (R.match(String: Name, Matches: &Groups))
1732 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1733 .Case(S: "add", Value: Intrinsic::vector_reduce_add)
1734 .Case(S: "mul", Value: Intrinsic::vector_reduce_mul)
1735 .Case(S: "and", Value: Intrinsic::vector_reduce_and)
1736 .Case(S: "or", Value: Intrinsic::vector_reduce_or)
1737 .Case(S: "xor", Value: Intrinsic::vector_reduce_xor)
1738 .Case(S: "smax", Value: Intrinsic::vector_reduce_smax)
1739 .Case(S: "smin", Value: Intrinsic::vector_reduce_smin)
1740 .Case(S: "umax", Value: Intrinsic::vector_reduce_umax)
1741 .Case(S: "umin", Value: Intrinsic::vector_reduce_umin)
1742 .Case(S: "fmax", Value: Intrinsic::vector_reduce_fmax)
1743 .Case(S: "fmin", Value: Intrinsic::vector_reduce_fmin)
1744 .Default(Value: Intrinsic::not_intrinsic);
1745
1746 bool V2 = false;
1747 if (ID == Intrinsic::not_intrinsic) {
1748 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1749 Groups.clear();
1750 V2 = true;
1751 if (R2.match(String: Name, Matches: &Groups))
1752 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1753 .Case(S: "fadd", Value: Intrinsic::vector_reduce_fadd)
1754 .Case(S: "fmul", Value: Intrinsic::vector_reduce_fmul)
1755 .Default(Value: Intrinsic::not_intrinsic);
1756 }
1757 if (ID != Intrinsic::not_intrinsic) {
1758 rename(GV: F);
1759 auto Args = F->getFunctionType()->params();
1760 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1761 OverloadTys: {Args[V2 ? 1 : 0]});
1762 return true;
1763 }
1764 break; // No other 'expermental.vector.reduce.*'.
1765 }
1766
1767 if (Name.consume_front(Prefix: "splice"))
1768 return true;
1769 break; // No other 'experimental.vector.*'.
1770 }
1771 if (Name.consume_front(Prefix: "experimental.stepvector.")) {
1772 Intrinsic::ID ID = Intrinsic::stepvector;
1773 rename(GV: F);
1774 NewFn = Intrinsic::getOrInsertDeclaration(
1775 M: F->getParent(), id: ID, OverloadTys: F->getFunctionType()->getReturnType());
1776 return true;
1777 }
1778 break; // No other 'e*'.
1779 case 'f':
1780 if (Name.starts_with(Prefix: "flt.rounds")) {
1781 rename(GV: F);
1782 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1783 id: Intrinsic::get_rounding);
1784 return true;
1785 }
1786 break;
1787 case 'i':
1788 if (Name.starts_with(Prefix: "invariant.group.barrier")) {
1789 // Rename invariant.group.barrier to launder.invariant.group
1790 auto Args = F->getFunctionType()->params();
1791 Type* ObjectPtr[1] = {Args[0]};
1792 rename(GV: F);
1793 NewFn = Intrinsic::getOrInsertDeclaration(
1794 M: F->getParent(), id: Intrinsic::launder_invariant_group, OverloadTys: ObjectPtr);
1795 return true;
1796 }
1797 break;
1798 case 'l': {
1799 bool IsLifetimeStart = Name.consume_front(Prefix: "lifetime.start");
1800 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front(Prefix: "lifetime.end");
1801 if (IsLifetimeStart || IsLifetimeEnd) {
1802 if (F->arg_size() == 2) {
1803 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1804 : Intrinsic::lifetime_end;
1805 rename(GV: F);
1806 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1807 // arguments. Use the Ptr argument to create new declaration.
1808 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
1809 OverloadTys: F->getArg(i: 1)->getType());
1810 return true;
1811 } else if (F->arg_size() == 1 && Name == ".i64") {
1812 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1813 // Autoupgrade prior to
1814 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1815 // intrinsic with no expected calls. To allow auto-upgrade process to
1816 // delete such invalid intrinsic declaration, set NewFn = nullptr
1817 // and return true here. If there are actual calls to this intrinsic
1818 // (which is not expected), they will be deleted in
1819 // UpgradeIntrinsicCall.
1820 NewFn = nullptr;
1821 return true;
1822 }
1823 }
1824 break;
1825 }
1826 case 'm': {
1827 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1828 // alignment parameter to embedding the alignment as an attribute of
1829 // the pointer args.
1830 if (unsigned ID = StringSwitch<unsigned>(Name)
1831 .StartsWith(S: "memcpy.", Value: Intrinsic::memcpy)
1832 .StartsWith(S: "memmove.", Value: Intrinsic::memmove)
1833 .Default(Value: 0)) {
1834 if (F->arg_size() == 5) {
1835 rename(GV: F);
1836 // Get the types of dest, src, and len
1837 ArrayRef<Type *> ParamTypes =
1838 F->getFunctionType()->params().slice(N: 0, M: 3);
1839 NewFn =
1840 Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: ParamTypes);
1841 return true;
1842 }
1843 }
1844 if (Name.starts_with(Prefix: "memset.") && F->arg_size() == 5) {
1845 rename(GV: F);
1846 // Get the types of dest, and len
1847 const auto *FT = F->getFunctionType();
1848 Type *ParamTypes[2] = {
1849 FT->getParamType(i: 0), // Dest
1850 FT->getParamType(i: 2) // len
1851 };
1852 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1853 id: Intrinsic::memset, OverloadTys: ParamTypes);
1854 return true;
1855 }
1856
1857 unsigned MaskedID =
1858 StringSwitch<unsigned>(Name)
1859 .StartsWith(S: "masked.load", Value: Intrinsic::masked_load)
1860 .StartsWith(S: "masked.gather", Value: Intrinsic::masked_gather)
1861 .StartsWith(S: "masked.store", Value: Intrinsic::masked_store)
1862 .StartsWith(S: "masked.scatter", Value: Intrinsic::masked_scatter)
1863 .Default(Value: 0);
1864 if (MaskedID && F->arg_size() == 4) {
1865 rename(GV: F);
1866 if (MaskedID == Intrinsic::masked_load ||
1867 MaskedID == Intrinsic::masked_gather) {
1868 NewFn = Intrinsic::getOrInsertDeclaration(
1869 M: F->getParent(), id: MaskedID,
1870 OverloadTys: {F->getReturnType(), F->getArg(i: 0)->getType()});
1871 return true;
1872 }
1873 NewFn = Intrinsic::getOrInsertDeclaration(
1874 M: F->getParent(), id: MaskedID,
1875 OverloadTys: {F->getArg(i: 0)->getType(), F->getArg(i: 1)->getType()});
1876 return true;
1877 }
1878 break;
1879 }
1880 case 'n': {
1881 if (Name.consume_front(Prefix: "nvvm.")) {
1882 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
1883 if (F->arg_size() == 1) {
1884 Intrinsic::ID IID =
1885 StringSwitch<Intrinsic::ID>(Name)
1886 .Cases(CaseStrings: {"brev32", "brev64"}, Value: Intrinsic::bitreverse)
1887 .Case(S: "clz.i", Value: Intrinsic::ctlz)
1888 .Case(S: "popc.i", Value: Intrinsic::ctpop)
1889 .Default(Value: Intrinsic::not_intrinsic);
1890 if (IID != Intrinsic::not_intrinsic) {
1891 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
1892 OverloadTys: {F->getReturnType()});
1893 return true;
1894 }
1895 } else if (F->arg_size() == 2) {
1896 Intrinsic::ID IID =
1897 StringSwitch<Intrinsic::ID>(Name)
1898 .Cases(CaseStrings: {"max.s", "max.i", "max.ll"}, Value: Intrinsic::smax)
1899 .Cases(CaseStrings: {"min.s", "min.i", "min.ll"}, Value: Intrinsic::smin)
1900 .Cases(CaseStrings: {"max.us", "max.ui", "max.ull"}, Value: Intrinsic::umax)
1901 .Cases(CaseStrings: {"min.us", "min.ui", "min.ull"}, Value: Intrinsic::umin)
1902 .Default(Value: Intrinsic::not_intrinsic);
1903 if (IID != Intrinsic::not_intrinsic) {
1904 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
1905 OverloadTys: {F->getReturnType()});
1906 return true;
1907 }
1908 }
1909
1910 // Check for nvvm intrinsics that need a return type adjustment.
1911 if (!F->getReturnType()->getScalarType()->isBFloatTy()) {
1912 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
1913 if (IID != Intrinsic::not_intrinsic) {
1914 NewFn = nullptr;
1915 return true;
1916 }
1917 }
1918
1919 // Upgrade Distributed Shared Memory Intrinsics
1920 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
1921 if (IID != Intrinsic::not_intrinsic) {
1922 rename(GV: F);
1923 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
1924 return true;
1925 }
1926
1927 // Upgrade TMA reduction intrinsics
1928 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
1929 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
1930 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
1931 if (IID != Intrinsic::not_intrinsic) {
1932 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
1933 return true;
1934 }
1935
1936 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
1937 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
1938 if (IID != Intrinsic::not_intrinsic) {
1939 rename(GV: F);
1940 NewFn = Intrinsic::getOrInsertDeclaration(
1941 M: F->getParent(), IID, RetTy: F->getReturnType(),
1942 ArgTys: F->getFunctionType()->params());
1943 return true;
1944 }
1945
1946 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
1947 // tcgen05.alloc shared variants to anyptr intrinsics.
1948 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
1949 if (IID != Intrinsic::not_intrinsic) {
1950 rename(GV: F);
1951 if (Intrinsic::isOverloaded(id: IID))
1952 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
1953 OverloadTys: {F->getArg(i: 0)->getType()});
1954 else
1955 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
1956 return true;
1957 }
1958
1959 // Upgrade TMA copy G2S Intrinsics
1960 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name);
1961 if (IID != Intrinsic::not_intrinsic) {
1962 rename(GV: F);
1963 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
1964 return true;
1965 }
1966
1967 // Upgrade tcgen05.mma intrinsics missing collector_usage_b.
1968 IID = shouldUpgradeNVPTXTcgen05MMAIntrinsic(F, Name);
1969 if (IID != Intrinsic::not_intrinsic) {
1970 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
1971 return NewFn != F;
1972 }
1973
1974 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
1975 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
1976 //
1977 // TODO: We could add lohi.i2d.
1978 bool Expand = false;
1979 if (Name.consume_front(Prefix: "abs."))
1980 // nvvm.abs.{i,ii}
1981 Expand =
1982 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
1983 else if (Name.consume_front(Prefix: "fabs."))
1984 // nvvm.fabs.{f,ftz.f,d}
1985 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
1986 else if (Name.consume_front(Prefix: "ex2.approx."))
1987 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
1988 Expand =
1989 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
1990 else if (Name.consume_front(Prefix: "atomic.load."))
1991 // nvvm.atomic.load.add.{f32,f64}.p
1992 // nvvm.atomic.load.{inc,dec}.32.p
1993 Expand = StringSwitch<bool>(Name)
1994 .StartsWith(S: "add.f32.p", Value: true)
1995 .StartsWith(S: "add.f64.p", Value: true)
1996 .StartsWith(S: "inc.32.p", Value: true)
1997 .StartsWith(S: "dec.32.p", Value: true)
1998 .Default(Value: false);
1999 else if (Name.consume_front(Prefix: "atomic."))
2000 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2001 // nvvm.atomic.cas.gen.i.{cta,sys}
2002 Expand = StringSwitch<bool>(Name)
2003 .StartsWith(S: "add.gen.", Value: true)
2004 .StartsWith(S: "exch.gen.", Value: true)
2005 .StartsWith(S: "max.gen.", Value: true)
2006 .StartsWith(S: "min.gen.", Value: true)
2007 .StartsWith(S: "inc.gen.", Value: true)
2008 .StartsWith(S: "dec.gen.", Value: true)
2009 .StartsWith(S: "and.gen.", Value: true)
2010 .StartsWith(S: "or.gen.", Value: true)
2011 .StartsWith(S: "xor.gen.", Value: true)
2012 .StartsWith(S: "cas.gen.", Value: true)
2013 .Default(Value: false);
2014 else if (Name.consume_front(Prefix: "bitcast."))
2015 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2016 Expand =
2017 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2018 else if (Name.consume_front(Prefix: "rotate."))
2019 // nvvm.rotate.{b32,b64,right.b64}
2020 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2021 else if (Name.consume_front(Prefix: "ptr.gen.to."))
2022 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2023 Expand = consumeNVVMPtrAddrSpace(Name);
2024 else if (Name.consume_front(Prefix: "ptr."))
2025 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2026 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(Prefix: ".to.gen");
2027 else if (Name.consume_front(Prefix: "ldg.global."))
2028 // nvvm.ldg.global.{i,p,f}
2029 Expand = (Name.starts_with(Prefix: "i.") || Name.starts_with(Prefix: "f.") ||
2030 Name.starts_with(Prefix: "p."));
2031 else
2032 Expand = StringSwitch<bool>(Name)
2033 .Case(S: "barrier0", Value: true)
2034 .Case(S: "barrier.n", Value: true)
2035 .Case(S: "barrier.sync.cnt", Value: true)
2036 .Case(S: "barrier.sync", Value: true)
2037 .Case(S: "barrier", Value: true)
2038 .Case(S: "bar.sync", Value: true)
2039 .Case(S: "barrier0.popc", Value: true)
2040 .Case(S: "barrier0.and", Value: true)
2041 .Case(S: "barrier0.or", Value: true)
2042 .Case(S: "clz.ll", Value: true)
2043 .Case(S: "popc.ll", Value: true)
2044 .Case(S: "h2f", Value: true)
2045 .Case(S: "swap.lo.hi.b64", Value: true)
2046 .Case(S: "tanh.approx.f32", Value: true)
2047 .Default(Value: false);
2048
2049 if (Expand) {
2050 NewFn = nullptr;
2051 return true;
2052 }
2053 break; // No other 'nvvm.*'.
2054 }
2055 break;
2056 }
2057 case 'o':
2058 if (Name.starts_with(Prefix: "objectsize.")) {
2059 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2060 if (F->arg_size() == 2 || F->arg_size() == 3) {
2061 rename(GV: F);
2062 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
2063 id: Intrinsic::objectsize, OverloadTys: Tys);
2064 return true;
2065 }
2066 }
2067 break;
2068
2069 case 'p':
2070 if (Name.starts_with(Prefix: "ptr.annotation.") && F->arg_size() == 4) {
2071 rename(GV: F);
2072 NewFn = Intrinsic::getOrInsertDeclaration(
2073 M: F->getParent(), id: Intrinsic::ptr_annotation,
2074 OverloadTys: {F->arg_begin()->getType(), F->getArg(i: 1)->getType()});
2075 return true;
2076 }
2077 break;
2078
2079 case 'r': {
2080 if (Name.consume_front(Prefix: "riscv.")) {
2081 Intrinsic::ID ID;
2082 ID = StringSwitch<Intrinsic::ID>(Name)
2083 .Case(S: "aes32dsi", Value: Intrinsic::riscv_aes32dsi)
2084 .Case(S: "aes32dsmi", Value: Intrinsic::riscv_aes32dsmi)
2085 .Case(S: "aes32esi", Value: Intrinsic::riscv_aes32esi)
2086 .Case(S: "aes32esmi", Value: Intrinsic::riscv_aes32esmi)
2087 .Default(Value: Intrinsic::not_intrinsic);
2088 if (ID != Intrinsic::not_intrinsic) {
2089 if (!F->getFunctionType()->getParamType(i: 2)->isIntegerTy(BitWidth: 32)) {
2090 rename(GV: F);
2091 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2092 return true;
2093 }
2094 break; // No other applicable upgrades.
2095 }
2096
2097 ID = StringSwitch<Intrinsic::ID>(Name)
2098 .StartsWith(S: "sm4ks", Value: Intrinsic::riscv_sm4ks)
2099 .StartsWith(S: "sm4ed", Value: Intrinsic::riscv_sm4ed)
2100 .Default(Value: Intrinsic::not_intrinsic);
2101 if (ID != Intrinsic::not_intrinsic) {
2102 if (!F->getFunctionType()->getParamType(i: 2)->isIntegerTy(BitWidth: 32) ||
2103 F->getFunctionType()->getReturnType()->isIntegerTy(BitWidth: 64)) {
2104 rename(GV: F);
2105 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2106 return true;
2107 }
2108 break; // No other applicable upgrades.
2109 }
2110
2111 ID = StringSwitch<Intrinsic::ID>(Name)
2112 .StartsWith(S: "sha256sig0", Value: Intrinsic::riscv_sha256sig0)
2113 .StartsWith(S: "sha256sig1", Value: Intrinsic::riscv_sha256sig1)
2114 .StartsWith(S: "sha256sum0", Value: Intrinsic::riscv_sha256sum0)
2115 .StartsWith(S: "sha256sum1", Value: Intrinsic::riscv_sha256sum1)
2116 .StartsWith(S: "sm3p0", Value: Intrinsic::riscv_sm3p0)
2117 .StartsWith(S: "sm3p1", Value: Intrinsic::riscv_sm3p1)
2118 .Default(Value: Intrinsic::not_intrinsic);
2119 if (ID != Intrinsic::not_intrinsic) {
2120 if (F->getFunctionType()->getReturnType()->isIntegerTy(BitWidth: 64)) {
2121 rename(GV: F);
2122 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2123 return true;
2124 }
2125 break; // No other applicable upgrades.
2126 }
2127
2128 // Replace llvm.riscv.clmul with llvm.clmul.
2129 if (Name == "clmul.i32" || Name == "clmul.i64") {
2130 NewFn = Intrinsic::getOrInsertDeclaration(
2131 M: F->getParent(), id: Intrinsic::clmul, OverloadTys: {F->getReturnType()});
2132 return true;
2133 }
2134
2135 break; // No other 'riscv.*' intrinsics
2136 }
2137 } break;
2138
2139 case 's':
2140 if (Name == "stackprotectorcheck") {
2141 NewFn = nullptr;
2142 return true;
2143 }
2144 break;
2145
2146 case 't':
2147 if (Name == "thread.pointer") {
2148 NewFn = Intrinsic::getOrInsertDeclaration(
2149 M: F->getParent(), id: Intrinsic::thread_pointer, OverloadTys: F->getReturnType());
2150 return true;
2151 }
2152 break;
2153
2154 case 'v': {
2155 if (Name == "var.annotation" && F->arg_size() == 4) {
2156 rename(GV: F);
2157 NewFn = Intrinsic::getOrInsertDeclaration(
2158 M: F->getParent(), id: Intrinsic::var_annotation,
2159 OverloadTys: {{F->arg_begin()->getType(), F->getArg(i: 1)->getType()}});
2160 return true;
2161 }
2162 if (Name.consume_front(Prefix: "vector.splice")) {
2163 if (Name.starts_with(Prefix: ".left") || Name.starts_with(Prefix: ".right"))
2164 break;
2165 return true;
2166 }
2167 if (shouldUpgradeVPIntrinsic(Name))
2168 return true;
2169 break;
2170 }
2171
2172 case 'w':
2173 if (Name.consume_front(Prefix: "wasm.")) {
2174 Intrinsic::ID ID =
2175 StringSwitch<Intrinsic::ID>(Name)
2176 .StartsWith(S: "fma.", Value: Intrinsic::wasm_relaxed_madd)
2177 .StartsWith(S: "fms.", Value: Intrinsic::wasm_relaxed_nmadd)
2178 .StartsWith(S: "laneselect.", Value: Intrinsic::wasm_relaxed_laneselect)
2179 .Default(Value: Intrinsic::not_intrinsic);
2180 if (ID != Intrinsic::not_intrinsic) {
2181 rename(GV: F);
2182 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
2183 OverloadTys: F->getReturnType());
2184 return true;
2185 }
2186
2187 if (Name.consume_front(Prefix: "dot.i8x16.i7x16.")) {
2188 ID = StringSwitch<Intrinsic::ID>(Name)
2189 .Case(S: "signed", Value: Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2190 .Case(S: "add.signed",
2191 Value: Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2192 .Default(Value: Intrinsic::not_intrinsic);
2193 if (ID != Intrinsic::not_intrinsic) {
2194 rename(GV: F);
2195 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2196 return true;
2197 }
2198 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2199 }
2200 break; // No other 'wasm.*'.
2201 }
2202 break;
2203
2204 case 'x':
2205 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2206 return true;
2207 }
2208
2209 auto *ST = dyn_cast<StructType>(Val: F->getReturnType());
2210 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2211 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2212 // Replace return type with literal non-packed struct. Only do this for
2213 // intrinsics declared to return a struct, not for intrinsics with
2214 // overloaded return type, in which case the exact struct type will be
2215 // mangled into the name.
2216 if (Intrinsic::hasStructReturnType(id: F->getIntrinsicID())) {
2217 FunctionType *FT = F->getFunctionType();
2218 auto *NewST = StructType::get(Context&: ST->getContext(), Elements: ST->elements());
2219 auto *NewFT = FunctionType::get(Result: NewST, Params: FT->params(), isVarArg: FT->isVarArg());
2220 std::string Name = F->getName().str();
2221 rename(GV: F);
2222 NewFn = Function::Create(Ty: NewFT, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
2223 N: Name, M: F->getParent());
2224
2225 // The new function may also need remangling.
2226 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F: NewFn))
2227 NewFn = *Result;
2228 return true;
2229 }
2230 }
2231
2232 // Remangle our intrinsic since we upgrade the mangling
2233 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2234 if (Result != std::nullopt) {
2235 NewFn = *Result;
2236 return true;
2237 }
2238
2239 // This may not belong here. This function is effectively being overloaded
2240 // to both detect an intrinsic which needs upgrading, and to provide the
2241 // upgraded form of the intrinsic. We should perhaps have two separate
2242 // functions for this.
2243 if (upgradeIntrinsicDeclWithDefaultArgs(F, NewFn))
2244 return true;
2245
2246 return false;
2247}
2248
2249bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2250 bool CanUpgradeDebugIntrinsicsToRecords) {
2251 NewFn = nullptr;
2252 bool Upgraded =
2253 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2254
2255 // Upgrade intrinsic attributes. This does not change the function.
2256 if (NewFn)
2257 F = NewFn;
2258 if (Intrinsic::ID id = F->getIntrinsicID()) {
2259 // Only do this if the intrinsic signature is valid.
2260 SmallVector<Type *> OverloadTys;
2261 if (Intrinsic::isSignatureValid(ID: id, FT: F->getFunctionType(), OverloadTys))
2262 F->setAttributes(
2263 Intrinsic::getAttributes(C&: F->getContext(), id, FT: F->getFunctionType()));
2264 }
2265 return Upgraded;
2266}
2267
2268GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2269 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2270 GV->getName() == "llvm.global_dtors")) ||
2271 !GV->hasInitializer())
2272 return nullptr;
2273 ArrayType *ATy = dyn_cast<ArrayType>(Val: GV->getValueType());
2274 if (!ATy)
2275 return nullptr;
2276 StructType *STy = dyn_cast<StructType>(Val: ATy->getElementType());
2277 if (!STy || STy->getNumElements() != 2)
2278 return nullptr;
2279
2280 LLVMContext &C = GV->getContext();
2281 IRBuilder<> IRB(C);
2282 auto EltTy = StructType::get(elt1: STy->getElementType(N: 0), elts: STy->getElementType(N: 1),
2283 elts: IRB.getPtrTy());
2284 Constant *Init = GV->getInitializer();
2285 unsigned N = Init->getNumOperands();
2286 std::vector<Constant *> NewCtors(N);
2287 for (unsigned i = 0; i != N; ++i) {
2288 auto Ctor = cast<Constant>(Val: Init->getOperand(i));
2289 NewCtors[i] = ConstantStruct::get(T: EltTy, Vs: Ctor->getAggregateElement(Elt: 0u),
2290 Vs: Ctor->getAggregateElement(Elt: 1),
2291 Vs: ConstantPointerNull::get(T: IRB.getPtrTy()));
2292 }
2293 Constant *NewInit = ConstantArray::get(T: ArrayType::get(ElementType: EltTy, NumElements: N), V: NewCtors);
2294
2295 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2296 NewInit, GV->getName());
2297}
2298
2299// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2300// to byte shuffles.
2301static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2302 unsigned Shift) {
2303 auto *ResultTy = cast<FixedVectorType>(Val: Op->getType());
2304 unsigned NumElts = ResultTy->getNumElements() * 8;
2305
2306 // Bitcast from a 64-bit element type to a byte element type.
2307 Type *VecTy = FixedVectorType::get(ElementType: Builder.getInt8Ty(), NumElts);
2308 Op = Builder.CreateBitCast(V: Op, DestTy: VecTy, Name: "cast");
2309
2310 // We'll be shuffling in zeroes.
2311 Value *Res = Constant::getNullValue(Ty: VecTy);
2312
2313 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2314 // we'll just return the zero vector.
2315 if (Shift < 16) {
2316 int Idxs[64];
2317 // 256/512-bit version is split into 2/4 16-byte lanes.
2318 for (unsigned l = 0; l != NumElts; l += 16)
2319 for (unsigned i = 0; i != 16; ++i) {
2320 unsigned Idx = NumElts + i - Shift;
2321 if (Idx < NumElts)
2322 Idx -= NumElts - 16; // end of lane, switch operand.
2323 Idxs[l + i] = Idx + l;
2324 }
2325
2326 Res = Builder.CreateShuffleVector(V1: Res, V2: Op, Mask: ArrayRef(Idxs, NumElts));
2327 }
2328
2329 // Bitcast back to a 64-bit element type.
2330 return Builder.CreateBitCast(V: Res, DestTy: ResultTy, Name: "cast");
2331}
2332
2333// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2334// to byte shuffles.
2335static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2336 unsigned Shift) {
2337 auto *ResultTy = cast<FixedVectorType>(Val: Op->getType());
2338 unsigned NumElts = ResultTy->getNumElements() * 8;
2339
2340 // Bitcast from a 64-bit element type to a byte element type.
2341 Type *VecTy = FixedVectorType::get(ElementType: Builder.getInt8Ty(), NumElts);
2342 Op = Builder.CreateBitCast(V: Op, DestTy: VecTy, Name: "cast");
2343
2344 // We'll be shuffling in zeroes.
2345 Value *Res = Constant::getNullValue(Ty: VecTy);
2346
2347 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2348 // we'll just return the zero vector.
2349 if (Shift < 16) {
2350 int Idxs[64];
2351 // 256/512-bit version is split into 2/4 16-byte lanes.
2352 for (unsigned l = 0; l != NumElts; l += 16)
2353 for (unsigned i = 0; i != 16; ++i) {
2354 unsigned Idx = i + Shift;
2355 if (Idx >= 16)
2356 Idx += NumElts - 16; // end of lane, switch operand.
2357 Idxs[l + i] = Idx + l;
2358 }
2359
2360 Res = Builder.CreateShuffleVector(V1: Op, V2: Res, Mask: ArrayRef(Idxs, NumElts));
2361 }
2362
2363 // Bitcast back to a 64-bit element type.
2364 return Builder.CreateBitCast(V: Res, DestTy: ResultTy, Name: "cast");
2365}
2366
2367static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2368 unsigned NumElts) {
2369 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2370 llvm::VectorType *MaskTy = FixedVectorType::get(
2371 ElementType: Builder.getInt1Ty(), NumElts: cast<IntegerType>(Val: Mask->getType())->getBitWidth());
2372 Mask = Builder.CreateBitCast(V: Mask, DestTy: MaskTy);
2373
2374 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2375 // i8 and we need to extract down to the right number of elements.
2376 if (NumElts <= 4) {
2377 int Indices[4];
2378 for (unsigned i = 0; i != NumElts; ++i)
2379 Indices[i] = i;
2380 Mask = Builder.CreateShuffleVector(V1: Mask, V2: Mask, Mask: ArrayRef(Indices, NumElts),
2381 Name: "extract");
2382 }
2383
2384 return Mask;
2385}
2386
2387static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2388 Value *Op1) {
2389 // If the mask is all ones just emit the first operation.
2390 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2391 if (C->isAllOnesValue())
2392 return Op0;
2393
2394 Mask = getX86MaskVec(Builder, Mask,
2395 NumElts: cast<FixedVectorType>(Val: Op0->getType())->getNumElements());
2396 return Builder.CreateSelect(C: Mask, True: Op0, False: Op1);
2397}
2398
2399static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2400 Value *Op1) {
2401 // If the mask is all ones just emit the first operation.
2402 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2403 if (C->isAllOnesValue())
2404 return Op0;
2405
2406 auto *MaskTy = FixedVectorType::get(ElementType: Builder.getInt1Ty(),
2407 NumElts: Mask->getType()->getIntegerBitWidth());
2408 Mask = Builder.CreateBitCast(V: Mask, DestTy: MaskTy);
2409 Mask = Builder.CreateExtractElement(Vec: Mask, Idx: (uint64_t)0);
2410 return Builder.CreateSelect(C: Mask, True: Op0, False: Op1);
2411}
2412
2413// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2414// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2415// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2416static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2417 Value *Op1, Value *Shift,
2418 Value *Passthru, Value *Mask,
2419 bool IsVALIGN) {
2420 unsigned ShiftVal = cast<llvm::ConstantInt>(Val: Shift)->getZExtValue();
2421
2422 unsigned NumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
2423 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2424 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2425 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2426
2427 // Mask the immediate for VALIGN.
2428 if (IsVALIGN)
2429 ShiftVal &= (NumElts - 1);
2430
2431 // If palignr is shifting the pair of vectors more than the size of two
2432 // lanes, emit zero.
2433 if (ShiftVal >= 32)
2434 return llvm::Constant::getNullValue(Ty: Op0->getType());
2435
2436 // If palignr is shifting the pair of input vectors more than one lane,
2437 // but less than two lanes, convert to shifting in zeroes.
2438 if (ShiftVal > 16) {
2439 ShiftVal -= 16;
2440 Op1 = Op0;
2441 Op0 = llvm::Constant::getNullValue(Ty: Op0->getType());
2442 }
2443
2444 int Indices[64];
2445 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2446 for (unsigned l = 0; l < NumElts; l += 16) {
2447 for (unsigned i = 0; i != 16; ++i) {
2448 unsigned Idx = ShiftVal + i;
2449 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2450 Idx += NumElts - 16; // End of lane, switch operand.
2451 Indices[l + i] = Idx + l;
2452 }
2453 }
2454
2455 Value *Align = Builder.CreateShuffleVector(
2456 V1: Op1, V2: Op0, Mask: ArrayRef(Indices, NumElts), Name: "palignr");
2457
2458 return emitX86Select(Builder, Mask, Op0: Align, Op1: Passthru);
2459}
2460
2461static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2462 bool ZeroMask, bool IndexForm) {
2463 Type *Ty = CI.getType();
2464 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2465 unsigned EltWidth = Ty->getScalarSizeInBits();
2466 bool IsFloat = Ty->isFPOrFPVectorTy();
2467 Intrinsic::ID IID;
2468 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2469 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2470 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2471 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2472 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2473 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2474 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2475 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2476 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2477 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2478 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2479 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2480 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2481 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2482 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2483 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2484 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2485 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2486 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2487 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2488 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2489 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2490 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2491 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2492 else if (VecWidth == 128 && EltWidth == 16)
2493 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2494 else if (VecWidth == 256 && EltWidth == 16)
2495 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2496 else if (VecWidth == 512 && EltWidth == 16)
2497 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2498 else if (VecWidth == 128 && EltWidth == 8)
2499 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2500 else if (VecWidth == 256 && EltWidth == 8)
2501 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2502 else if (VecWidth == 512 && EltWidth == 8)
2503 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2504 else
2505 llvm_unreachable("Unexpected intrinsic");
2506
2507 Value *Args[] = { CI.getArgOperand(i: 0) , CI.getArgOperand(i: 1),
2508 CI.getArgOperand(i: 2) };
2509
2510 // If this isn't index form we need to swap operand 0 and 1.
2511 if (!IndexForm)
2512 std::swap(a&: Args[0], b&: Args[1]);
2513
2514 Value *V = Builder.CreateIntrinsic(ID: IID, Args);
2515 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2516 : Builder.CreateBitCast(V: CI.getArgOperand(i: 1),
2517 DestTy: Ty);
2518 return emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: V, Op1: PassThru);
2519}
2520
2521static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2522 Intrinsic::ID IID) {
2523 Type *Ty = CI.getType();
2524 Value *Op0 = CI.getOperand(i_nocapture: 0);
2525 Value *Op1 = CI.getOperand(i_nocapture: 1);
2526 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Op0, Op1});
2527
2528 if (CI.arg_size() == 4) { // For masked intrinsics.
2529 Value *VecSrc = CI.getOperand(i_nocapture: 2);
2530 Value *Mask = CI.getOperand(i_nocapture: 3);
2531 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2532 }
2533 return Res;
2534}
2535
2536static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2537 bool IsRotateRight) {
2538 Type *Ty = CI.getType();
2539 Value *Src = CI.getArgOperand(i: 0);
2540 Value *Amt = CI.getArgOperand(i: 1);
2541
2542 // Amount may be scalar immediate, in which case create a splat vector.
2543 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2544 // we only care about the lowest log2 bits anyway.
2545 if (Amt->getType() != Ty) {
2546 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
2547 Amt = Builder.CreateIntCast(V: Amt, DestTy: Ty->getScalarType(), isSigned: false);
2548 Amt = Builder.CreateVectorSplat(NumElts, V: Amt);
2549 }
2550
2551 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2552 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Src, Src, Amt});
2553
2554 if (CI.arg_size() == 4) { // For masked intrinsics.
2555 Value *VecSrc = CI.getOperand(i_nocapture: 2);
2556 Value *Mask = CI.getOperand(i_nocapture: 3);
2557 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2558 }
2559 return Res;
2560}
2561
2562static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2563 bool IsSigned) {
2564 Type *Ty = CI.getType();
2565 Value *LHS = CI.getArgOperand(i: 0);
2566 Value *RHS = CI.getArgOperand(i: 1);
2567
2568 CmpInst::Predicate Pred;
2569 switch (Imm) {
2570 case 0x0:
2571 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2572 break;
2573 case 0x1:
2574 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2575 break;
2576 case 0x2:
2577 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2578 break;
2579 case 0x3:
2580 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2581 break;
2582 case 0x4:
2583 Pred = ICmpInst::ICMP_EQ;
2584 break;
2585 case 0x5:
2586 Pred = ICmpInst::ICMP_NE;
2587 break;
2588 case 0x6:
2589 return Constant::getNullValue(Ty); // FALSE
2590 case 0x7:
2591 return Constant::getAllOnesValue(Ty); // TRUE
2592 default:
2593 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2594 }
2595
2596 Value *Cmp = Builder.CreateICmp(P: Pred, LHS, RHS);
2597 Value *Ext = Builder.CreateSExt(V: Cmp, DestTy: Ty);
2598 return Ext;
2599}
2600
2601static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2602 bool IsShiftRight, bool ZeroMask) {
2603 Type *Ty = CI.getType();
2604 Value *Op0 = CI.getArgOperand(i: 0);
2605 Value *Op1 = CI.getArgOperand(i: 1);
2606 Value *Amt = CI.getArgOperand(i: 2);
2607
2608 if (IsShiftRight)
2609 std::swap(a&: Op0, b&: Op1);
2610
2611 // Amount may be scalar immediate, in which case create a splat vector.
2612 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2613 // we only care about the lowest log2 bits anyway.
2614 if (Amt->getType() != Ty) {
2615 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
2616 Amt = Builder.CreateIntCast(V: Amt, DestTy: Ty->getScalarType(), isSigned: false);
2617 Amt = Builder.CreateVectorSplat(NumElts, V: Amt);
2618 }
2619
2620 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2621 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Op0, Op1, Amt});
2622
2623 unsigned NumArgs = CI.arg_size();
2624 if (NumArgs >= 4) { // For masked intrinsics.
2625 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(i: 3) :
2626 ZeroMask ? ConstantAggregateZero::get(Ty: CI.getType()) :
2627 CI.getArgOperand(i: 0);
2628 Value *Mask = CI.getOperand(i_nocapture: NumArgs - 1);
2629 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2630 }
2631 return Res;
2632}
2633
2634static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2635 Value *Mask, bool Aligned) {
2636 const Align Alignment =
2637 Aligned
2638 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2639 : Align(1);
2640
2641 // If the mask is all ones just emit a regular store.
2642 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2643 if (C->isAllOnesValue())
2644 return Builder.CreateAlignedStore(Val: Data, Ptr, Align: Alignment);
2645
2646 // Convert the mask from an integer type to a vector of i1.
2647 unsigned NumElts = cast<FixedVectorType>(Val: Data->getType())->getNumElements();
2648 Mask = getX86MaskVec(Builder, Mask, NumElts);
2649 return Builder.CreateMaskedStore(Val: Data, Ptr, Alignment, Mask);
2650}
2651
2652static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2653 Value *Passthru, Value *Mask, bool Aligned) {
2654 Type *ValTy = Passthru->getType();
2655 const Align Alignment =
2656 Aligned
2657 ? Align(
2658 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2659 8)
2660 : Align(1);
2661
2662 // If the mask is all ones just emit a regular store.
2663 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2664 if (C->isAllOnesValue())
2665 return Builder.CreateAlignedLoad(Ty: ValTy, Ptr, Align: Alignment);
2666
2667 // Convert the mask from an integer type to a vector of i1.
2668 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
2669 Mask = getX86MaskVec(Builder, Mask, NumElts);
2670 return Builder.CreateMaskedLoad(Ty: ValTy, Ptr, Alignment, Mask, PassThru: Passthru);
2671}
2672
2673static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2674 Type *Ty = CI.getType();
2675 Value *Op0 = CI.getArgOperand(i: 0);
2676 Value *Res = Builder.CreateIntrinsic(ID: Intrinsic::abs, OverloadTypes: Ty,
2677 Args: {Op0, Builder.getInt1(V: false)});
2678 if (CI.arg_size() == 3)
2679 Res = emitX86Select(Builder, Mask: CI.getArgOperand(i: 2), Op0: Res, Op1: CI.getArgOperand(i: 1));
2680 return Res;
2681}
2682
2683static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2684 Type *Ty = CI.getType();
2685
2686 // Arguments have a vXi32 type so cast to vXi64.
2687 Value *LHS = Builder.CreateBitCast(V: CI.getArgOperand(i: 0), DestTy: Ty);
2688 Value *RHS = Builder.CreateBitCast(V: CI.getArgOperand(i: 1), DestTy: Ty);
2689
2690 if (IsSigned) {
2691 // Shift left then arithmetic shift right.
2692 Constant *ShiftAmt = ConstantInt::get(Ty, V: 32);
2693 LHS = Builder.CreateShl(LHS, RHS: ShiftAmt);
2694 LHS = Builder.CreateAShr(LHS, RHS: ShiftAmt);
2695 RHS = Builder.CreateShl(LHS: RHS, RHS: ShiftAmt);
2696 RHS = Builder.CreateAShr(LHS: RHS, RHS: ShiftAmt);
2697 } else {
2698 // Clear the upper bits.
2699 Constant *Mask = ConstantInt::get(Ty, V: 0xffffffff);
2700 LHS = Builder.CreateAnd(LHS, RHS: Mask);
2701 RHS = Builder.CreateAnd(LHS: RHS, RHS: Mask);
2702 }
2703
2704 Value *Res = Builder.CreateMul(LHS, RHS);
2705
2706 if (CI.arg_size() == 4)
2707 Res = emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: Res, Op1: CI.getArgOperand(i: 2));
2708
2709 return Res;
2710}
2711
2712// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2713static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2714 Value *Mask) {
2715 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2716 if (Mask) {
2717 const auto *C = dyn_cast<Constant>(Val: Mask);
2718 if (!C || !C->isAllOnesValue())
2719 Vec = Builder.CreateAnd(LHS: Vec, RHS: getX86MaskVec(Builder, Mask, NumElts));
2720 }
2721
2722 if (NumElts < 8) {
2723 int Indices[8];
2724 for (unsigned i = 0; i != NumElts; ++i)
2725 Indices[i] = i;
2726 for (unsigned i = NumElts; i != 8; ++i)
2727 Indices[i] = NumElts + i % NumElts;
2728 Vec = Builder.CreateShuffleVector(V1: Vec,
2729 V2: Constant::getNullValue(Ty: Vec->getType()),
2730 Mask: Indices);
2731 }
2732 return Builder.CreateBitCast(V: Vec, DestTy: Builder.getIntNTy(N: std::max(a: NumElts, b: 8U)));
2733}
2734
2735static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2736 unsigned CC, bool Signed) {
2737 Value *Op0 = CI.getArgOperand(i: 0);
2738 unsigned NumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
2739
2740 Value *Cmp;
2741 if (CC == 3) {
2742 Cmp = Constant::getNullValue(
2743 Ty: FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts));
2744 } else if (CC == 7) {
2745 Cmp = Constant::getAllOnesValue(
2746 Ty: FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts));
2747 } else {
2748 ICmpInst::Predicate Pred;
2749 switch (CC) {
2750 default: llvm_unreachable("Unknown condition code");
2751 case 0: Pred = ICmpInst::ICMP_EQ; break;
2752 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2753 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2754 case 4: Pred = ICmpInst::ICMP_NE; break;
2755 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2756 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2757 }
2758 Cmp = Builder.CreateICmp(P: Pred, LHS: Op0, RHS: CI.getArgOperand(i: 1));
2759 }
2760
2761 Value *Mask = CI.getArgOperand(i: CI.arg_size() - 1);
2762
2763 return applyX86MaskOn1BitsVec(Builder, Vec: Cmp, Mask);
2764}
2765
2766// Replace a masked intrinsic with an older unmasked intrinsic.
2767static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2768 Intrinsic::ID IID) {
2769 Value *Rep =
2770 Builder.CreateIntrinsic(ID: IID, Args: {CI.getArgOperand(i: 0), CI.getArgOperand(i: 1)});
2771 return emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: Rep, Op1: CI.getArgOperand(i: 2));
2772}
2773
2774static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2775 Value* A = CI.getArgOperand(i: 0);
2776 Value* B = CI.getArgOperand(i: 1);
2777 Value* Src = CI.getArgOperand(i: 2);
2778 Value* Mask = CI.getArgOperand(i: 3);
2779
2780 Value* AndNode = Builder.CreateAnd(LHS: Mask, RHS: APInt(8, 1));
2781 Value* Cmp = Builder.CreateIsNotNull(Arg: AndNode);
2782 Value* Extract1 = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0);
2783 Value* Extract2 = Builder.CreateExtractElement(Vec: Src, Idx: (uint64_t)0);
2784 Value* Select = Builder.CreateSelect(C: Cmp, True: Extract1, False: Extract2);
2785 return Builder.CreateInsertElement(Vec: A, NewElt: Select, Idx: (uint64_t)0);
2786}
2787
2788static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2789 Value* Op = CI.getArgOperand(i: 0);
2790 Type* ReturnOp = CI.getType();
2791 unsigned NumElts = cast<FixedVectorType>(Val: CI.getType())->getNumElements();
2792 Value *Mask = getX86MaskVec(Builder, Mask: Op, NumElts);
2793 return Builder.CreateSExt(V: Mask, DestTy: ReturnOp, Name: "vpmovm2");
2794}
2795
2796// Replace intrinsic with unmasked version and a select.
2797static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2798 CallBase &CI, Value *&Rep) {
2799 Name = Name.substr(Start: 12); // Remove avx512.mask.
2800
2801 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2802 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2803 Intrinsic::ID IID;
2804 if (Name.starts_with(Prefix: "max.p")) {
2805 if (VecWidth == 128 && EltWidth == 32)
2806 IID = Intrinsic::x86_sse_max_ps;
2807 else if (VecWidth == 128 && EltWidth == 64)
2808 IID = Intrinsic::x86_sse2_max_pd;
2809 else if (VecWidth == 256 && EltWidth == 32)
2810 IID = Intrinsic::x86_avx_max_ps_256;
2811 else if (VecWidth == 256 && EltWidth == 64)
2812 IID = Intrinsic::x86_avx_max_pd_256;
2813 else
2814 llvm_unreachable("Unexpected intrinsic");
2815 } else if (Name.starts_with(Prefix: "min.p")) {
2816 if (VecWidth == 128 && EltWidth == 32)
2817 IID = Intrinsic::x86_sse_min_ps;
2818 else if (VecWidth == 128 && EltWidth == 64)
2819 IID = Intrinsic::x86_sse2_min_pd;
2820 else if (VecWidth == 256 && EltWidth == 32)
2821 IID = Intrinsic::x86_avx_min_ps_256;
2822 else if (VecWidth == 256 && EltWidth == 64)
2823 IID = Intrinsic::x86_avx_min_pd_256;
2824 else
2825 llvm_unreachable("Unexpected intrinsic");
2826 } else if (Name.starts_with(Prefix: "pshuf.b.")) {
2827 if (VecWidth == 128)
2828 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2829 else if (VecWidth == 256)
2830 IID = Intrinsic::x86_avx2_pshuf_b;
2831 else if (VecWidth == 512)
2832 IID = Intrinsic::x86_avx512_pshuf_b_512;
2833 else
2834 llvm_unreachable("Unexpected intrinsic");
2835 } else if (Name.starts_with(Prefix: "pmul.hr.sw.")) {
2836 if (VecWidth == 128)
2837 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2838 else if (VecWidth == 256)
2839 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2840 else if (VecWidth == 512)
2841 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2842 else
2843 llvm_unreachable("Unexpected intrinsic");
2844 } else if (Name.starts_with(Prefix: "pmulh.w.")) {
2845 if (VecWidth == 128)
2846 IID = Intrinsic::x86_sse2_pmulh_w;
2847 else if (VecWidth == 256)
2848 IID = Intrinsic::x86_avx2_pmulh_w;
2849 else if (VecWidth == 512)
2850 IID = Intrinsic::x86_avx512_pmulh_w_512;
2851 else
2852 llvm_unreachable("Unexpected intrinsic");
2853 } else if (Name.starts_with(Prefix: "pmulhu.w.")) {
2854 if (VecWidth == 128)
2855 IID = Intrinsic::x86_sse2_pmulhu_w;
2856 else if (VecWidth == 256)
2857 IID = Intrinsic::x86_avx2_pmulhu_w;
2858 else if (VecWidth == 512)
2859 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2860 else
2861 llvm_unreachable("Unexpected intrinsic");
2862 } else if (Name.starts_with(Prefix: "pmaddw.d.")) {
2863 if (VecWidth == 128)
2864 IID = Intrinsic::x86_sse2_pmadd_wd;
2865 else if (VecWidth == 256)
2866 IID = Intrinsic::x86_avx2_pmadd_wd;
2867 else if (VecWidth == 512)
2868 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2869 else
2870 llvm_unreachable("Unexpected intrinsic");
2871 } else if (Name.starts_with(Prefix: "pmaddubs.w.")) {
2872 if (VecWidth == 128)
2873 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2874 else if (VecWidth == 256)
2875 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2876 else if (VecWidth == 512)
2877 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2878 else
2879 llvm_unreachable("Unexpected intrinsic");
2880 } else if (Name.starts_with(Prefix: "packsswb.")) {
2881 if (VecWidth == 128)
2882 IID = Intrinsic::x86_sse2_packsswb_128;
2883 else if (VecWidth == 256)
2884 IID = Intrinsic::x86_avx2_packsswb;
2885 else if (VecWidth == 512)
2886 IID = Intrinsic::x86_avx512_packsswb_512;
2887 else
2888 llvm_unreachable("Unexpected intrinsic");
2889 } else if (Name.starts_with(Prefix: "packssdw.")) {
2890 if (VecWidth == 128)
2891 IID = Intrinsic::x86_sse2_packssdw_128;
2892 else if (VecWidth == 256)
2893 IID = Intrinsic::x86_avx2_packssdw;
2894 else if (VecWidth == 512)
2895 IID = Intrinsic::x86_avx512_packssdw_512;
2896 else
2897 llvm_unreachable("Unexpected intrinsic");
2898 } else if (Name.starts_with(Prefix: "packuswb.")) {
2899 if (VecWidth == 128)
2900 IID = Intrinsic::x86_sse2_packuswb_128;
2901 else if (VecWidth == 256)
2902 IID = Intrinsic::x86_avx2_packuswb;
2903 else if (VecWidth == 512)
2904 IID = Intrinsic::x86_avx512_packuswb_512;
2905 else
2906 llvm_unreachable("Unexpected intrinsic");
2907 } else if (Name.starts_with(Prefix: "packusdw.")) {
2908 if (VecWidth == 128)
2909 IID = Intrinsic::x86_sse41_packusdw;
2910 else if (VecWidth == 256)
2911 IID = Intrinsic::x86_avx2_packusdw;
2912 else if (VecWidth == 512)
2913 IID = Intrinsic::x86_avx512_packusdw_512;
2914 else
2915 llvm_unreachable("Unexpected intrinsic");
2916 } else if (Name.starts_with(Prefix: "vpermilvar.")) {
2917 if (VecWidth == 128 && EltWidth == 32)
2918 IID = Intrinsic::x86_avx_vpermilvar_ps;
2919 else if (VecWidth == 128 && EltWidth == 64)
2920 IID = Intrinsic::x86_avx_vpermilvar_pd;
2921 else if (VecWidth == 256 && EltWidth == 32)
2922 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2923 else if (VecWidth == 256 && EltWidth == 64)
2924 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2925 else if (VecWidth == 512 && EltWidth == 32)
2926 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2927 else if (VecWidth == 512 && EltWidth == 64)
2928 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2929 else
2930 llvm_unreachable("Unexpected intrinsic");
2931 } else if (Name == "cvtpd2dq.256") {
2932 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2933 } else if (Name == "cvtpd2ps.256") {
2934 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2935 } else if (Name == "cvttpd2dq.256") {
2936 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2937 } else if (Name == "cvttps2dq.128") {
2938 IID = Intrinsic::x86_sse2_cvttps2dq;
2939 } else if (Name == "cvttps2dq.256") {
2940 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2941 } else if (Name.starts_with(Prefix: "permvar.")) {
2942 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
2943 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2944 IID = Intrinsic::x86_avx2_permps;
2945 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2946 IID = Intrinsic::x86_avx2_permd;
2947 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2948 IID = Intrinsic::x86_avx512_permvar_df_256;
2949 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2950 IID = Intrinsic::x86_avx512_permvar_di_256;
2951 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2952 IID = Intrinsic::x86_avx512_permvar_sf_512;
2953 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2954 IID = Intrinsic::x86_avx512_permvar_si_512;
2955 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2956 IID = Intrinsic::x86_avx512_permvar_df_512;
2957 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2958 IID = Intrinsic::x86_avx512_permvar_di_512;
2959 else if (VecWidth == 128 && EltWidth == 16)
2960 IID = Intrinsic::x86_avx512_permvar_hi_128;
2961 else if (VecWidth == 256 && EltWidth == 16)
2962 IID = Intrinsic::x86_avx512_permvar_hi_256;
2963 else if (VecWidth == 512 && EltWidth == 16)
2964 IID = Intrinsic::x86_avx512_permvar_hi_512;
2965 else if (VecWidth == 128 && EltWidth == 8)
2966 IID = Intrinsic::x86_avx512_permvar_qi_128;
2967 else if (VecWidth == 256 && EltWidth == 8)
2968 IID = Intrinsic::x86_avx512_permvar_qi_256;
2969 else if (VecWidth == 512 && EltWidth == 8)
2970 IID = Intrinsic::x86_avx512_permvar_qi_512;
2971 else
2972 llvm_unreachable("Unexpected intrinsic");
2973 } else if (Name.starts_with(Prefix: "dbpsadbw.")) {
2974 if (VecWidth == 128)
2975 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2976 else if (VecWidth == 256)
2977 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2978 else if (VecWidth == 512)
2979 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2980 else
2981 llvm_unreachable("Unexpected intrinsic");
2982 } else if (Name.starts_with(Prefix: "pmultishift.qb.")) {
2983 if (VecWidth == 128)
2984 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2985 else if (VecWidth == 256)
2986 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2987 else if (VecWidth == 512)
2988 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2989 else
2990 llvm_unreachable("Unexpected intrinsic");
2991 } else if (Name.starts_with(Prefix: "conflict.")) {
2992 if (Name[9] == 'd' && VecWidth == 128)
2993 IID = Intrinsic::x86_avx512_conflict_d_128;
2994 else if (Name[9] == 'd' && VecWidth == 256)
2995 IID = Intrinsic::x86_avx512_conflict_d_256;
2996 else if (Name[9] == 'd' && VecWidth == 512)
2997 IID = Intrinsic::x86_avx512_conflict_d_512;
2998 else if (Name[9] == 'q' && VecWidth == 128)
2999 IID = Intrinsic::x86_avx512_conflict_q_128;
3000 else if (Name[9] == 'q' && VecWidth == 256)
3001 IID = Intrinsic::x86_avx512_conflict_q_256;
3002 else if (Name[9] == 'q' && VecWidth == 512)
3003 IID = Intrinsic::x86_avx512_conflict_q_512;
3004 else
3005 llvm_unreachable("Unexpected intrinsic");
3006 } else if (Name.starts_with(Prefix: "pavg.")) {
3007 if (Name[5] == 'b' && VecWidth == 128)
3008 IID = Intrinsic::x86_sse2_pavg_b;
3009 else if (Name[5] == 'b' && VecWidth == 256)
3010 IID = Intrinsic::x86_avx2_pavg_b;
3011 else if (Name[5] == 'b' && VecWidth == 512)
3012 IID = Intrinsic::x86_avx512_pavg_b_512;
3013 else if (Name[5] == 'w' && VecWidth == 128)
3014 IID = Intrinsic::x86_sse2_pavg_w;
3015 else if (Name[5] == 'w' && VecWidth == 256)
3016 IID = Intrinsic::x86_avx2_pavg_w;
3017 else if (Name[5] == 'w' && VecWidth == 512)
3018 IID = Intrinsic::x86_avx512_pavg_w_512;
3019 else
3020 llvm_unreachable("Unexpected intrinsic");
3021 } else
3022 return false;
3023
3024 SmallVector<Value *, 4> Args(CI.args());
3025 Args.pop_back();
3026 Args.pop_back();
3027 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3028 unsigned NumArgs = CI.arg_size();
3029 Rep = emitX86Select(Builder, Mask: CI.getArgOperand(i: NumArgs - 1), Op0: Rep,
3030 Op1: CI.getArgOperand(i: NumArgs - 2));
3031 return true;
3032}
3033
3034/// Upgrade comment in call to inline asm that represents an objc retain release
3035/// marker.
3036void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3037 size_t Pos;
3038 if (AsmStr->find(s: "mov\tfp") == 0 &&
3039 AsmStr->find(s: "objc_retainAutoreleaseReturnValue") != std::string::npos &&
3040 (Pos = AsmStr->find(s: "# marker")) != std::string::npos) {
3041 AsmStr->replace(pos: Pos, n1: 1, s: ";");
3042 }
3043}
3044
3045static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3046 Function *F, IRBuilder<> &Builder) {
3047 Value *Rep = nullptr;
3048
3049 if (Name == "abs.i" || Name == "abs.ll") {
3050 Value *Arg = CI->getArgOperand(i: 0);
3051 Rep = Builder.CreateIntrinsic(ID: Intrinsic::abs, OverloadTypes: {Arg->getType()},
3052 Args: {Arg, Builder.getTrue()},
3053 /*FMFSource=*/nullptr, Name: "abs");
3054 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3055 Type *Ty = (Name == "abs.bf16")
3056 ? Builder.getBFloatTy()
3057 : FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 2);
3058 Value *Arg = Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Ty);
3059 Value *Abs = Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_fabs, Op: Arg);
3060 Rep = Builder.CreateBitCast(V: Abs, DestTy: CI->getType());
3061 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3062 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3063 : Intrinsic::nvvm_fabs;
3064 Rep = Builder.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0));
3065 } else if (Name.consume_front(Prefix: "ex2.approx.")) {
3066 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3067 Intrinsic::ID IID = Name.starts_with(Prefix: "ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3068 : Intrinsic::nvvm_ex2_approx;
3069 Rep = Builder.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0));
3070 } else if (Name.starts_with(Prefix: "atomic.load.add.f32.p") ||
3071 Name.starts_with(Prefix: "atomic.load.add.f64.p")) {
3072 Value *Ptr = CI->getArgOperand(i: 0);
3073 Value *Val = CI->getArgOperand(i: 1);
3074 Rep = Builder.CreateAtomicRMW(
3075 Op: AtomicRMWInst::FAdd, Ptr, Val, Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic,
3076 SSID: CI->getContext().getOrInsertSyncScopeID(SSN: "device"));
3077 // The default scope for atomic.load.* intrinsics is device
3078 // (= gpu scope in ptx), but the default LLVM atomic scope is
3079 // "system"
3080 } else if (Name.starts_with(Prefix: "atomic.load.inc.32.p") ||
3081 Name.starts_with(Prefix: "atomic.load.dec.32.p")) {
3082 Value *Ptr = CI->getArgOperand(i: 0);
3083 Value *Val = CI->getArgOperand(i: 1);
3084 auto Op = Name.starts_with(Prefix: "atomic.load.inc") ? AtomicRMWInst::UIncWrap
3085 : AtomicRMWInst::UDecWrap;
3086 Rep = Builder.CreateAtomicRMW(
3087 Op, Ptr, Val, Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic,
3088 SSID: CI->getContext().getOrInsertSyncScopeID(SSN: "device"));
3089 // See comment above.
3090 } else if (Name.starts_with(Prefix: "atomic.") && Name.contains(Other: ".gen.")) {
3091 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3092 StringRef Op = Name.substr(Start: StringRef("atomic.").size());
3093 Value *Ptr = CI->getArgOperand(i: 0);
3094 Value *Val = CI->getArgOperand(i: 1);
3095 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3096 SSN: Op.contains(Other: ".cta.") ? "block" : "");
3097 if (Op.starts_with(Prefix: "cas.")) {
3098 Value *New = CI->getArgOperand(i: 2);
3099 Value *Pair = Builder.CreateAtomicCmpXchg(
3100 Ptr, Cmp: Val, New, Align: MaybeAlign(), SuccessOrdering: AtomicOrdering::Monotonic,
3101 FailureOrdering: AtomicOrdering::Monotonic, SSID);
3102 Rep = Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
3103 } else {
3104 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3105 // because we were actually missing those intrinsics!
3106 AtomicRMWInst::BinOp BinOp =
3107 StringSwitch<AtomicRMWInst::BinOp>(Op)
3108 .StartsWith(S: "add.gen.f", Value: AtomicRMWInst::FAdd)
3109 .StartsWith(S: "add.gen.i", Value: AtomicRMWInst::Add)
3110 .StartsWith(S: "exch.", Value: AtomicRMWInst::Xchg)
3111 .StartsWith(S: "max.", Value: AtomicRMWInst::Max)
3112 .StartsWith(S: "min.", Value: AtomicRMWInst::Min)
3113 .StartsWith(S: "inc.", Value: AtomicRMWInst::UIncWrap)
3114 .StartsWith(S: "dec.", Value: AtomicRMWInst::UDecWrap)
3115 .StartsWith(S: "and.", Value: AtomicRMWInst::And)
3116 .StartsWith(S: "or.", Value: AtomicRMWInst::Or)
3117 .StartsWith(S: "xor.", Value: AtomicRMWInst::Xor)
3118 .Default(Value: AtomicRMWInst::BAD_BINOP);
3119 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3120 "unexpected nvvm scoped atomic intrinsic");
3121 Rep = Builder.CreateAtomicRMW(Op: BinOp, Ptr, Val, Align: MaybeAlign(),
3122 Ordering: AtomicOrdering::Monotonic, SSID);
3123 }
3124 } else if (Name == "clz.ll") {
3125 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3126 Value *Arg = CI->getArgOperand(i: 0);
3127 Value *Ctlz = Builder.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: {Arg->getType()},
3128 Args: {Arg, Builder.getFalse()},
3129 /*FMFSource=*/nullptr, Name: "ctlz");
3130 Rep = Builder.CreateTrunc(V: Ctlz, DestTy: Builder.getInt32Ty(), Name: "ctlz.trunc");
3131 } else if (Name == "popc.ll") {
3132 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3133 // i64.
3134 Value *Arg = CI->getArgOperand(i: 0);
3135 Value *Popc = Builder.CreateIntrinsic(ID: Intrinsic::ctpop, OverloadTypes: {Arg->getType()},
3136 Args: Arg, /*FMFSource=*/nullptr, Name: "ctpop");
3137 Rep = Builder.CreateTrunc(V: Popc, DestTy: Builder.getInt32Ty(), Name: "ctpop.trunc");
3138 } else if (Name == "h2f") {
3139 Value *Cast =
3140 Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
3141 Rep = Builder.CreateFPExt(V: Cast, DestTy: Builder.getFloatTy());
3142 } else if (Name.consume_front(Prefix: "bitcast.") &&
3143 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3144 Name == "d2ll")) {
3145 Rep = Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: CI->getType());
3146 } else if (Name == "rotate.b32") {
3147 Value *Arg = CI->getOperand(i_nocapture: 0);
3148 Value *ShiftAmt = CI->getOperand(i_nocapture: 1);
3149 Rep = Builder.CreateIntrinsic(RetTy: Builder.getInt32Ty(), ID: Intrinsic::fshl,
3150 Args: {Arg, Arg, ShiftAmt});
3151 } else if (Name == "rotate.b64") {
3152 Type *Int64Ty = Builder.getInt64Ty();
3153 Value *Arg = CI->getOperand(i_nocapture: 0);
3154 Value *ZExtShiftAmt = Builder.CreateZExt(V: CI->getOperand(i_nocapture: 1), DestTy: Int64Ty);
3155 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshl,
3156 Args: {Arg, Arg, ZExtShiftAmt});
3157 } else if (Name == "rotate.right.b64") {
3158 Type *Int64Ty = Builder.getInt64Ty();
3159 Value *Arg = CI->getOperand(i_nocapture: 0);
3160 Value *ZExtShiftAmt = Builder.CreateZExt(V: CI->getOperand(i_nocapture: 1), DestTy: Int64Ty);
3161 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshr,
3162 Args: {Arg, Arg, ZExtShiftAmt});
3163 } else if (Name == "swap.lo.hi.b64") {
3164 Type *Int64Ty = Builder.getInt64Ty();
3165 Value *Arg = CI->getOperand(i_nocapture: 0);
3166 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshl,
3167 Args: {Arg, Arg, Builder.getInt64(C: 32)});
3168 } else if ((Name.consume_front(Prefix: "ptr.gen.to.") &&
3169 consumeNVVMPtrAddrSpace(Name)) ||
3170 (Name.consume_front(Prefix: "ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3171 Name.starts_with(Prefix: ".to.gen"))) {
3172 Rep = Builder.CreateAddrSpaceCast(V: CI->getArgOperand(i: 0), DestTy: CI->getType());
3173 } else if (Name.consume_front(Prefix: "ldg.global")) {
3174 Value *Ptr = CI->getArgOperand(i: 0);
3175 Align PtrAlign = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getAlignValue();
3176 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3177 Value *ASC = Builder.CreateAddrSpaceCast(V: Ptr, DestTy: Builder.getPtrTy(AddrSpace: 1));
3178 Instruction *LD = Builder.CreateAlignedLoad(Ty: CI->getType(), Ptr: ASC, Align: PtrAlign);
3179 MDNode *MD = MDNode::get(Context&: Builder.getContext(), MDs: {});
3180 LD->setMetadata(KindID: LLVMContext::MD_invariant_load, Node: MD);
3181 return LD;
3182 } else if (Name == "tanh.approx.f32") {
3183 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3184 FastMathFlags FMF;
3185 FMF.setApproxFunc();
3186 Rep = Builder.CreateUnaryIntrinsic(ID: Intrinsic::tanh, Op: CI->getArgOperand(i: 0),
3187 FMFSource: FMF);
3188 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3189 Value *Arg =
3190 Name.ends_with(Suffix: '0') ? Builder.getInt32(C: 0) : CI->getArgOperand(i: 0);
3191 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3192 OverloadTypes: {}, Args: {Arg});
3193 } else if (Name == "barrier") {
3194 Rep = Builder.CreateIntrinsic(
3195 ID: Intrinsic::nvvm_barrier_cta_sync_aligned_count, OverloadTypes: {},
3196 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
3197 } else if (Name == "barrier.sync") {
3198 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_all, OverloadTypes: {},
3199 Args: {CI->getArgOperand(i: 0)});
3200 } else if (Name == "barrier.sync.cnt") {
3201 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_count, OverloadTypes: {},
3202 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
3203 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3204 Name == "barrier0.or") {
3205 Value *C = CI->getArgOperand(i: 0);
3206 C = Builder.CreateICmpNE(LHS: C, RHS: Builder.getInt32(C: 0));
3207
3208 Intrinsic::ID IID =
3209 StringSwitch<Intrinsic::ID>(Name)
3210 .Case(S: "barrier0.popc",
3211 Value: Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3212 .Case(S: "barrier0.and",
3213 Value: Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3214 .Case(S: "barrier0.or",
3215 Value: Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3216 Value *Bar = Builder.CreateIntrinsic(ID: IID, OverloadTypes: {}, Args: {Builder.getInt32(C: 0), C});
3217 Rep = Builder.CreateZExt(V: Bar, DestTy: CI->getType());
3218 } else {
3219 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3220 if (IID != Intrinsic::not_intrinsic &&
3221 !F->getReturnType()->getScalarType()->isBFloatTy()) {
3222 rename(GV: F);
3223 Function *NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
3224 SmallVector<Value *, 2> Args;
3225 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3226 Value *Arg = CI->getArgOperand(i: I);
3227 Type *OldType = Arg->getType();
3228 Type *NewType = NewFn->getArg(i: I)->getType();
3229 Args.push_back(
3230 Elt: (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3231 ? Builder.CreateBitCast(V: Arg, DestTy: NewType)
3232 : Arg);
3233 }
3234 Rep = Builder.CreateCall(Callee: NewFn, Args);
3235 if (F->getReturnType()->isIntegerTy())
3236 Rep = Builder.CreateBitCast(V: Rep, DestTy: F->getReturnType());
3237 }
3238 }
3239
3240 return Rep;
3241}
3242
3243static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3244 IRBuilder<> &Builder) {
3245 LLVMContext &C = F->getContext();
3246 Value *Rep = nullptr;
3247
3248 if (Name.starts_with(Prefix: "sse4a.movnt.")) {
3249 SmallVector<Metadata *, 1> Elts;
3250 Elts.push_back(
3251 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
3252 MDNode *Node = MDNode::get(Context&: C, MDs: Elts);
3253
3254 Value *Arg0 = CI->getArgOperand(i: 0);
3255 Value *Arg1 = CI->getArgOperand(i: 1);
3256
3257 // Nontemporal (unaligned) store of the 0'th element of the float/double
3258 // vector.
3259 Value *Extract =
3260 Builder.CreateExtractElement(Vec: Arg1, Idx: (uint64_t)0, Name: "extractelement");
3261
3262 StoreInst *SI = Builder.CreateAlignedStore(Val: Extract, Ptr: Arg0, Align: Align(1));
3263 SI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
3264 } else if (Name.starts_with(Prefix: "avx.movnt.") ||
3265 Name.starts_with(Prefix: "avx512.storent.")) {
3266 SmallVector<Metadata *, 1> Elts;
3267 Elts.push_back(
3268 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
3269 MDNode *Node = MDNode::get(Context&: C, MDs: Elts);
3270
3271 Value *Arg0 = CI->getArgOperand(i: 0);
3272 Value *Arg1 = CI->getArgOperand(i: 1);
3273
3274 StoreInst *SI = Builder.CreateAlignedStore(
3275 Val: Arg1, Ptr: Arg0,
3276 Align: Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3277 SI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
3278 } else if (Name == "sse2.storel.dq") {
3279 Value *Arg0 = CI->getArgOperand(i: 0);
3280 Value *Arg1 = CI->getArgOperand(i: 1);
3281
3282 auto *NewVecTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C), NumElts: 2);
3283 Value *BC0 = Builder.CreateBitCast(V: Arg1, DestTy: NewVecTy, Name: "cast");
3284 Value *Elt = Builder.CreateExtractElement(Vec: BC0, Idx: (uint64_t)0);
3285 Builder.CreateAlignedStore(Val: Elt, Ptr: Arg0, Align: Align(1));
3286 } else if (Name.starts_with(Prefix: "sse.storeu.") ||
3287 Name.starts_with(Prefix: "sse2.storeu.") ||
3288 Name.starts_with(Prefix: "avx.storeu.")) {
3289 Value *Arg0 = CI->getArgOperand(i: 0);
3290 Value *Arg1 = CI->getArgOperand(i: 1);
3291 Builder.CreateAlignedStore(Val: Arg1, Ptr: Arg0, Align: Align(1));
3292 } else if (Name == "avx512.mask.store.ss") {
3293 Value *Mask = Builder.CreateAnd(LHS: CI->getArgOperand(i: 2), RHS: Builder.getInt8(C: 1));
3294 upgradeMaskedStore(Builder, Ptr: CI->getArgOperand(i: 0), Data: CI->getArgOperand(i: 1),
3295 Mask, Aligned: false);
3296 } else if (Name.starts_with(Prefix: "avx512.mask.store")) {
3297 // "avx512.mask.storeu." or "avx512.mask.store."
3298 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3299 upgradeMaskedStore(Builder, Ptr: CI->getArgOperand(i: 0), Data: CI->getArgOperand(i: 1),
3300 Mask: CI->getArgOperand(i: 2), Aligned);
3301 } else if (Name.starts_with(Prefix: "sse2.pcmp") || Name.starts_with(Prefix: "avx2.pcmp")) {
3302 // Upgrade packed integer vector compare intrinsics to compare instructions.
3303 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3304 bool CmpEq = Name[9] == 'e';
3305 Rep = Builder.CreateICmp(P: CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3306 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
3307 Rep = Builder.CreateSExt(V: Rep, DestTy: CI->getType(), Name: "");
3308 } else if (Name.starts_with(Prefix: "avx512.broadcastm")) {
3309 Type *ExtTy = Type::getInt32Ty(C);
3310 if (CI->getOperand(i_nocapture: 0)->getType()->isIntegerTy(BitWidth: 8))
3311 ExtTy = Type::getInt64Ty(C);
3312 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3313 ExtTy->getPrimitiveSizeInBits();
3314 Rep = Builder.CreateZExt(V: CI->getArgOperand(i: 0), DestTy: ExtTy);
3315 Rep = Builder.CreateVectorSplat(NumElts, V: Rep);
3316 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3317 Value *Vec = CI->getArgOperand(i: 0);
3318 Value *Elt0 = Builder.CreateExtractElement(Vec, Idx: (uint64_t)0);
3319 Elt0 = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: Elt0->getType(), Args: Elt0);
3320 Rep = Builder.CreateInsertElement(Vec, NewElt: Elt0, Idx: (uint64_t)0);
3321 } else if (Name.starts_with(Prefix: "avx.sqrt.p") ||
3322 Name.starts_with(Prefix: "sse2.sqrt.p") ||
3323 Name.starts_with(Prefix: "sse.sqrt.p")) {
3324 Rep = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: CI->getType(),
3325 Args: {CI->getArgOperand(i: 0)});
3326 } else if (Name.starts_with(Prefix: "avx512.mask.sqrt.p")) {
3327 if (CI->arg_size() == 4 &&
3328 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 3)) ||
3329 cast<ConstantInt>(Val: CI->getArgOperand(i: 3))->getZExtValue() != 4)) {
3330 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3331 : Intrinsic::x86_avx512_sqrt_pd_512;
3332
3333 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 3)};
3334 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3335 } else {
3336 Rep = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: CI->getType(),
3337 Args: {CI->getArgOperand(i: 0)});
3338 }
3339 Rep =
3340 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3341 } else if (Name.starts_with(Prefix: "avx512.ptestm") ||
3342 Name.starts_with(Prefix: "avx512.ptestnm")) {
3343 Value *Op0 = CI->getArgOperand(i: 0);
3344 Value *Op1 = CI->getArgOperand(i: 1);
3345 Value *Mask = CI->getArgOperand(i: 2);
3346 Rep = Builder.CreateAnd(LHS: Op0, RHS: Op1);
3347 llvm::Type *Ty = Op0->getType();
3348 Value *Zero = llvm::Constant::getNullValue(Ty);
3349 ICmpInst::Predicate Pred = Name.starts_with(Prefix: "avx512.ptestm")
3350 ? ICmpInst::ICMP_NE
3351 : ICmpInst::ICMP_EQ;
3352 Rep = Builder.CreateICmp(P: Pred, LHS: Rep, RHS: Zero);
3353 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask);
3354 } else if (Name.starts_with(Prefix: "avx512.mask.pbroadcast")) {
3355 unsigned NumElts = cast<FixedVectorType>(Val: CI->getArgOperand(i: 1)->getType())
3356 ->getNumElements();
3357 Rep = Builder.CreateVectorSplat(NumElts, V: CI->getArgOperand(i: 0));
3358 Rep =
3359 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3360 } else if (Name.starts_with(Prefix: "avx512.kunpck")) {
3361 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3362 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts);
3363 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts);
3364 int Indices[64];
3365 for (unsigned i = 0; i != NumElts; ++i)
3366 Indices[i] = i;
3367
3368 // First extract half of each vector. This gives better codegen than
3369 // doing it in a single shuffle.
3370 LHS = Builder.CreateShuffleVector(V1: LHS, V2: LHS, Mask: ArrayRef(Indices, NumElts / 2));
3371 RHS = Builder.CreateShuffleVector(V1: RHS, V2: RHS, Mask: ArrayRef(Indices, NumElts / 2));
3372 // Concat the vectors.
3373 // NOTE: Operands have to be swapped to match intrinsic definition.
3374 Rep = Builder.CreateShuffleVector(V1: RHS, V2: LHS, Mask: ArrayRef(Indices, NumElts));
3375 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3376 } else if (Name == "avx512.kand.w") {
3377 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3378 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3379 Rep = Builder.CreateAnd(LHS, RHS);
3380 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3381 } else if (Name == "avx512.kandn.w") {
3382 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3383 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3384 LHS = Builder.CreateNot(V: LHS);
3385 Rep = Builder.CreateAnd(LHS, RHS);
3386 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3387 } else if (Name == "avx512.kor.w") {
3388 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3389 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3390 Rep = Builder.CreateOr(LHS, RHS);
3391 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3392 } else if (Name == "avx512.kxor.w") {
3393 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3394 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3395 Rep = Builder.CreateXor(LHS, RHS);
3396 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3397 } else if (Name == "avx512.kxnor.w") {
3398 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3399 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3400 LHS = Builder.CreateNot(V: LHS);
3401 Rep = Builder.CreateXor(LHS, RHS);
3402 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3403 } else if (Name == "avx512.knot.w") {
3404 Rep = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3405 Rep = Builder.CreateNot(V: Rep);
3406 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3407 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3408 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3409 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3410 Rep = Builder.CreateOr(LHS, RHS);
3411 Rep = Builder.CreateBitCast(V: Rep, DestTy: Builder.getInt16Ty());
3412 Value *C;
3413 if (Name[14] == 'c')
3414 C = ConstantInt::getAllOnesValue(Ty: Builder.getInt16Ty());
3415 else
3416 C = ConstantInt::getNullValue(Ty: Builder.getInt16Ty());
3417 Rep = Builder.CreateICmpEQ(LHS: Rep, RHS: C);
3418 Rep = Builder.CreateZExt(V: Rep, DestTy: Builder.getInt32Ty());
3419 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3420 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3421 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3422 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3423 Type *I32Ty = Type::getInt32Ty(C);
3424 Value *Elt0 = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 0),
3425 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3426 Value *Elt1 = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 1),
3427 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3428 Value *EltOp;
3429 if (Name.contains(Other: ".add."))
3430 EltOp = Builder.CreateFAdd(L: Elt0, R: Elt1);
3431 else if (Name.contains(Other: ".sub."))
3432 EltOp = Builder.CreateFSub(L: Elt0, R: Elt1);
3433 else if (Name.contains(Other: ".mul."))
3434 EltOp = Builder.CreateFMul(L: Elt0, R: Elt1);
3435 else
3436 EltOp = Builder.CreateFDiv(L: Elt0, R: Elt1);
3437 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: EltOp,
3438 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3439 } else if (Name.starts_with(Prefix: "avx512.mask.pcmp")) {
3440 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3441 bool CmpEq = Name[16] == 'e';
3442 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: CmpEq ? 0 : 6, Signed: true);
3443 } else if (Name.starts_with(Prefix: "avx512.mask.vpshufbitqmb.")) {
3444 Type *OpTy = CI->getArgOperand(i: 0)->getType();
3445 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3446 Intrinsic::ID IID;
3447 switch (VecWidth) {
3448 default:
3449 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3450 break;
3451 case 128:
3452 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3453 break;
3454 case 256:
3455 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3456 break;
3457 case 512:
3458 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3459 break;
3460 }
3461
3462 Rep =
3463 Builder.CreateIntrinsic(ID: IID, Args: {CI->getOperand(i_nocapture: 0), CI->getArgOperand(i: 1)});
3464 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: CI->getArgOperand(i: 2));
3465 } else if (Name.starts_with(Prefix: "avx512.mask.fpclass.p")) {
3466 Type *OpTy = CI->getArgOperand(i: 0)->getType();
3467 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3468 unsigned EltWidth = OpTy->getScalarSizeInBits();
3469 Intrinsic::ID IID;
3470 if (VecWidth == 128 && EltWidth == 32)
3471 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3472 else if (VecWidth == 256 && EltWidth == 32)
3473 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3474 else if (VecWidth == 512 && EltWidth == 32)
3475 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3476 else if (VecWidth == 128 && EltWidth == 64)
3477 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3478 else if (VecWidth == 256 && EltWidth == 64)
3479 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3480 else if (VecWidth == 512 && EltWidth == 64)
3481 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3482 else
3483 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3484
3485 Rep =
3486 Builder.CreateIntrinsic(ID: IID, Args: {CI->getOperand(i_nocapture: 0), CI->getArgOperand(i: 1)});
3487 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: CI->getArgOperand(i: 2));
3488 } else if (Name.starts_with(Prefix: "avx512.cmp.p")) {
3489 SmallVector<Value *, 4> Args(CI->args());
3490 Type *OpTy = Args[0]->getType();
3491 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3492 unsigned EltWidth = OpTy->getScalarSizeInBits();
3493 Intrinsic::ID IID;
3494 if (VecWidth == 128 && EltWidth == 32)
3495 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3496 else if (VecWidth == 256 && EltWidth == 32)
3497 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3498 else if (VecWidth == 512 && EltWidth == 32)
3499 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3500 else if (VecWidth == 128 && EltWidth == 64)
3501 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3502 else if (VecWidth == 256 && EltWidth == 64)
3503 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3504 else if (VecWidth == 512 && EltWidth == 64)
3505 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3506 else
3507 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3508
3509 Value *Mask = Constant::getAllOnesValue(Ty: CI->getType());
3510 if (VecWidth == 512)
3511 std::swap(a&: Mask, b&: Args.back());
3512 Args.push_back(Elt: Mask);
3513
3514 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3515 } else if (Name.starts_with(Prefix: "avx512.mask.cmp.")) {
3516 // Integer compare intrinsics.
3517 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3518 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: Imm, Signed: true);
3519 } else if (Name.starts_with(Prefix: "avx512.mask.ucmp.")) {
3520 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3521 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: Imm, Signed: false);
3522 } else if (Name.starts_with(Prefix: "avx512.cvtb2mask.") ||
3523 Name.starts_with(Prefix: "avx512.cvtw2mask.") ||
3524 Name.starts_with(Prefix: "avx512.cvtd2mask.") ||
3525 Name.starts_with(Prefix: "avx512.cvtq2mask.")) {
3526 Value *Op = CI->getArgOperand(i: 0);
3527 Value *Zero = llvm::Constant::getNullValue(Ty: Op->getType());
3528 Rep = Builder.CreateICmp(P: ICmpInst::ICMP_SLT, LHS: Op, RHS: Zero);
3529 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: nullptr);
3530 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3531 Name == "ssse3.pabs.d.128" || Name.starts_with(Prefix: "avx2.pabs") ||
3532 Name.starts_with(Prefix: "avx512.mask.pabs")) {
3533 Rep = upgradeAbs(Builder, CI&: *CI);
3534 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3535 Name == "sse41.pmaxsd" || Name.starts_with(Prefix: "avx2.pmaxs") ||
3536 Name.starts_with(Prefix: "avx512.mask.pmaxs")) {
3537 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smax);
3538 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3539 Name == "sse41.pmaxud" || Name.starts_with(Prefix: "avx2.pmaxu") ||
3540 Name.starts_with(Prefix: "avx512.mask.pmaxu")) {
3541 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umax);
3542 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3543 Name == "sse41.pminsd" || Name.starts_with(Prefix: "avx2.pmins") ||
3544 Name.starts_with(Prefix: "avx512.mask.pmins")) {
3545 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smin);
3546 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3547 Name == "sse41.pminud" || Name.starts_with(Prefix: "avx2.pminu") ||
3548 Name.starts_with(Prefix: "avx512.mask.pminu")) {
3549 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umin);
3550 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3551 Name == "avx512.pmulu.dq.512" ||
3552 Name.starts_with(Prefix: "avx512.mask.pmulu.dq.")) {
3553 Rep = upgradePMULDQ(Builder, CI&: *CI, /*Signed*/ IsSigned: false);
3554 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3555 Name == "avx512.pmul.dq.512" ||
3556 Name.starts_with(Prefix: "avx512.mask.pmul.dq.")) {
3557 Rep = upgradePMULDQ(Builder, CI&: *CI, /*Signed*/ IsSigned: true);
3558 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3559 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3560 Rep =
3561 Builder.CreateSIToFP(V: CI->getArgOperand(i: 1),
3562 DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3563 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3564 } else if (Name == "avx512.cvtusi2sd") {
3565 Rep =
3566 Builder.CreateUIToFP(V: CI->getArgOperand(i: 1),
3567 DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3568 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3569 } else if (Name == "sse2.cvtss2sd") {
3570 Rep = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 1), Idx: (uint64_t)0);
3571 Rep = Builder.CreateFPExt(
3572 V: Rep, DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3573 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3574 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3575 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3576 Name.starts_with(Prefix: "avx512.mask.cvtdq2pd.") ||
3577 Name.starts_with(Prefix: "avx512.mask.cvtudq2pd.") ||
3578 Name.starts_with(Prefix: "avx512.mask.cvtdq2ps.") ||
3579 Name.starts_with(Prefix: "avx512.mask.cvtudq2ps.") ||
3580 Name.starts_with(Prefix: "avx512.mask.cvtqq2pd.") ||
3581 Name.starts_with(Prefix: "avx512.mask.cvtuqq2pd.") ||
3582 Name == "avx512.mask.cvtqq2ps.256" ||
3583 Name == "avx512.mask.cvtqq2ps.512" ||
3584 Name == "avx512.mask.cvtuqq2ps.256" ||
3585 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3586 Name == "avx.cvt.ps2.pd.256" ||
3587 Name == "avx512.mask.cvtps2pd.128" ||
3588 Name == "avx512.mask.cvtps2pd.256") {
3589 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3590 Rep = CI->getArgOperand(i: 0);
3591 auto *SrcTy = cast<FixedVectorType>(Val: Rep->getType());
3592
3593 unsigned NumDstElts = DstTy->getNumElements();
3594 if (NumDstElts < SrcTy->getNumElements()) {
3595 assert(NumDstElts == 2 && "Unexpected vector size");
3596 Rep = Builder.CreateShuffleVector(V1: Rep, V2: Rep, Mask: ArrayRef<int>{0, 1});
3597 }
3598
3599 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3600 bool IsUnsigned = Name.contains(Other: "cvtu");
3601 if (IsPS2PD)
3602 Rep = Builder.CreateFPExt(V: Rep, DestTy: DstTy, Name: "cvtps2pd");
3603 else if (CI->arg_size() == 4 &&
3604 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 3)) ||
3605 cast<ConstantInt>(Val: CI->getArgOperand(i: 3))->getZExtValue() != 4)) {
3606 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3607 : Intrinsic::x86_avx512_sitofp_round;
3608 Rep = Builder.CreateIntrinsic(ID: IID, OverloadTypes: {DstTy, SrcTy},
3609 Args: {Rep, CI->getArgOperand(i: 3)});
3610 } else {
3611 Rep = IsUnsigned ? Builder.CreateUIToFP(V: Rep, DestTy: DstTy, Name: "cvt")
3612 : Builder.CreateSIToFP(V: Rep, DestTy: DstTy, Name: "cvt");
3613 }
3614
3615 if (CI->arg_size() >= 3)
3616 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3617 Op1: CI->getArgOperand(i: 1));
3618 } else if (Name.starts_with(Prefix: "avx512.mask.vcvtph2ps.") ||
3619 Name.starts_with(Prefix: "vcvtph2ps.")) {
3620 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3621 Rep = CI->getArgOperand(i: 0);
3622 auto *SrcTy = cast<FixedVectorType>(Val: Rep->getType());
3623 unsigned NumDstElts = DstTy->getNumElements();
3624 if (NumDstElts != SrcTy->getNumElements()) {
3625 assert(NumDstElts == 4 && "Unexpected vector size");
3626 Rep = Builder.CreateShuffleVector(V1: Rep, V2: Rep, Mask: ArrayRef<int>{0, 1, 2, 3});
3627 }
3628 Rep = Builder.CreateBitCast(
3629 V: Rep, DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C), NumElts: NumDstElts));
3630 Rep = Builder.CreateFPExt(V: Rep, DestTy: DstTy, Name: "cvtph2ps");
3631 if (CI->arg_size() >= 3)
3632 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3633 Op1: CI->getArgOperand(i: 1));
3634 } else if (Name.starts_with(Prefix: "avx512.mask.load")) {
3635 // "avx512.mask.loadu." or "avx512.mask.load."
3636 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3637 Rep = upgradeMaskedLoad(Builder, Ptr: CI->getArgOperand(i: 0), Passthru: CI->getArgOperand(i: 1),
3638 Mask: CI->getArgOperand(i: 2), Aligned);
3639 } else if (Name.starts_with(Prefix: "avx512.mask.expand.load.")) {
3640 auto *ResultTy = cast<FixedVectorType>(Val: CI->getType());
3641 auto *PtrTy = CI->getOperand(i_nocapture: 0)->getType();
3642 Value *MaskVec = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3643 NumElts: ResultTy->getNumElements());
3644 Rep = Builder.CreateIntrinsic(
3645 ID: Intrinsic::masked_expandload, OverloadTypes: {ResultTy, PtrTy},
3646 Args: {CI->getOperand(i_nocapture: 0), MaskVec, CI->getOperand(i_nocapture: 1)});
3647 } else if (Name.starts_with(Prefix: "avx512.mask.compress.store.")) {
3648 auto *ResultTy = cast<VectorType>(Val: CI->getArgOperand(i: 1)->getType());
3649 auto *PtrTy = CI->getArgOperand(i: 0)->getType();
3650 Value *MaskVec =
3651 getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3652 NumElts: cast<FixedVectorType>(Val: ResultTy)->getNumElements());
3653 Rep = Builder.CreateIntrinsic(
3654 ID: Intrinsic::masked_compressstore, OverloadTypes: {ResultTy, PtrTy},
3655 Args: {CI->getArgOperand(i: 1), CI->getArgOperand(i: 0), MaskVec});
3656 } else if (Name.starts_with(Prefix: "avx512.mask.compress.") ||
3657 Name.starts_with(Prefix: "avx512.mask.expand.")) {
3658 auto *ResultTy = cast<FixedVectorType>(Val: CI->getType());
3659
3660 Value *MaskVec = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3661 NumElts: ResultTy->getNumElements());
3662
3663 bool IsCompress = Name[12] == 'c';
3664 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3665 : Intrinsic::x86_avx512_mask_expand;
3666 Rep = Builder.CreateIntrinsic(
3667 ID: IID, OverloadTypes: ResultTy, Args: {CI->getOperand(i_nocapture: 0), CI->getOperand(i_nocapture: 1), MaskVec});
3668 } else if (Name.starts_with(Prefix: "xop.vpcom")) {
3669 bool IsSigned;
3670 if (Name.ends_with(Suffix: "ub") || Name.ends_with(Suffix: "uw") || Name.ends_with(Suffix: "ud") ||
3671 Name.ends_with(Suffix: "uq"))
3672 IsSigned = false;
3673 else if (Name.ends_with(Suffix: "b") || Name.ends_with(Suffix: "w") ||
3674 Name.ends_with(Suffix: "d") || Name.ends_with(Suffix: "q"))
3675 IsSigned = true;
3676 else
3677 reportFatalUsageErrorWithCI(reason: "Intrinsic has unknown suffix", CI);
3678
3679 unsigned Imm;
3680 if (CI->arg_size() == 3) {
3681 Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3682 } else {
3683 Name = Name.substr(Start: 9); // strip off "xop.vpcom"
3684 if (Name.starts_with(Prefix: "lt"))
3685 Imm = 0;
3686 else if (Name.starts_with(Prefix: "le"))
3687 Imm = 1;
3688 else if (Name.starts_with(Prefix: "gt"))
3689 Imm = 2;
3690 else if (Name.starts_with(Prefix: "ge"))
3691 Imm = 3;
3692 else if (Name.starts_with(Prefix: "eq"))
3693 Imm = 4;
3694 else if (Name.starts_with(Prefix: "ne"))
3695 Imm = 5;
3696 else if (Name.starts_with(Prefix: "false"))
3697 Imm = 6;
3698 else if (Name.starts_with(Prefix: "true"))
3699 Imm = 7;
3700 else
3701 llvm_unreachable("Unknown condition");
3702 }
3703
3704 Rep = upgradeX86vpcom(Builder, CI&: *CI, Imm, IsSigned);
3705 } else if (Name.starts_with(Prefix: "xop.vpcmov")) {
3706 Value *Sel = CI->getArgOperand(i: 2);
3707 Value *NotSel = Builder.CreateNot(V: Sel);
3708 Value *Sel0 = Builder.CreateAnd(LHS: CI->getArgOperand(i: 0), RHS: Sel);
3709 Value *Sel1 = Builder.CreateAnd(LHS: CI->getArgOperand(i: 1), RHS: NotSel);
3710 Rep = Builder.CreateOr(LHS: Sel0, RHS: Sel1);
3711 } else if (Name.starts_with(Prefix: "xop.vprot") || Name.starts_with(Prefix: "avx512.prol") ||
3712 Name.starts_with(Prefix: "avx512.mask.prol")) {
3713 Rep = upgradeX86Rotate(Builder, CI&: *CI, IsRotateRight: false);
3714 } else if (Name.starts_with(Prefix: "avx512.pror") ||
3715 Name.starts_with(Prefix: "avx512.mask.pror")) {
3716 Rep = upgradeX86Rotate(Builder, CI&: *CI, IsRotateRight: true);
3717 } else if (Name.starts_with(Prefix: "avx512.vpshld.") ||
3718 Name.starts_with(Prefix: "avx512.mask.vpshld") ||
3719 Name.starts_with(Prefix: "avx512.maskz.vpshld")) {
3720 bool ZeroMask = Name[11] == 'z';
3721 Rep = upgradeX86ConcatShift(Builder, CI&: *CI, IsShiftRight: false, ZeroMask);
3722 } else if (Name.starts_with(Prefix: "avx512.vpshrd.") ||
3723 Name.starts_with(Prefix: "avx512.mask.vpshrd") ||
3724 Name.starts_with(Prefix: "avx512.maskz.vpshrd")) {
3725 bool ZeroMask = Name[11] == 'z';
3726 Rep = upgradeX86ConcatShift(Builder, CI&: *CI, IsShiftRight: true, ZeroMask);
3727 } else if (Name == "sse42.crc32.64.8") {
3728 Value *Trunc0 =
3729 Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Type::getInt32Ty(C));
3730 Rep = Builder.CreateIntrinsic(ID: Intrinsic::x86_sse42_crc32_32_8,
3731 Args: {Trunc0, CI->getArgOperand(i: 1)});
3732 Rep = Builder.CreateZExt(V: Rep, DestTy: CI->getType(), Name: "");
3733 } else if (Name.starts_with(Prefix: "avx.vbroadcast.s") ||
3734 Name.starts_with(Prefix: "avx512.vbroadcast.s")) {
3735 // Replace broadcasts with a series of insertelements.
3736 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
3737 Type *EltTy = VecTy->getElementType();
3738 unsigned EltNum = VecTy->getNumElements();
3739 Value *Load = Builder.CreateLoad(Ty: EltTy, Ptr: CI->getArgOperand(i: 0));
3740 Type *I32Ty = Type::getInt32Ty(C);
3741 Rep = PoisonValue::get(T: VecTy);
3742 for (unsigned I = 0; I < EltNum; ++I)
3743 Rep = Builder.CreateInsertElement(Vec: Rep, NewElt: Load, Idx: ConstantInt::get(Ty: I32Ty, V: I));
3744 } else if (Name.starts_with(Prefix: "sse41.pmovsx") ||
3745 Name.starts_with(Prefix: "sse41.pmovzx") ||
3746 Name.starts_with(Prefix: "avx2.pmovsx") ||
3747 Name.starts_with(Prefix: "avx2.pmovzx") ||
3748 Name.starts_with(Prefix: "avx512.mask.pmovsx") ||
3749 Name.starts_with(Prefix: "avx512.mask.pmovzx")) {
3750 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3751 unsigned NumDstElts = DstTy->getNumElements();
3752
3753 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3754 SmallVector<int, 8> ShuffleMask(NumDstElts);
3755 for (unsigned i = 0; i != NumDstElts; ++i)
3756 ShuffleMask[i] = i;
3757
3758 Value *SV = Builder.CreateShuffleVector(V: CI->getArgOperand(i: 0), Mask: ShuffleMask);
3759
3760 bool DoSext = Name.contains(Other: "pmovsx");
3761 Rep =
3762 DoSext ? Builder.CreateSExt(V: SV, DestTy: DstTy) : Builder.CreateZExt(V: SV, DestTy: DstTy);
3763 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3764 if (CI->arg_size() == 3)
3765 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3766 Op1: CI->getArgOperand(i: 1));
3767 } else if (Name == "avx512.mask.pmov.qd.256" ||
3768 Name == "avx512.mask.pmov.qd.512" ||
3769 Name == "avx512.mask.pmov.wb.256" ||
3770 Name == "avx512.mask.pmov.wb.512") {
3771 Type *Ty = CI->getArgOperand(i: 1)->getType();
3772 Rep = Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Ty);
3773 Rep =
3774 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3775 } else if (Name.starts_with(Prefix: "avx.vbroadcastf128") ||
3776 Name == "avx2.vbroadcasti128") {
3777 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3778 Type *EltTy = cast<VectorType>(Val: CI->getType())->getElementType();
3779 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3780 auto *VT = FixedVectorType::get(ElementType: EltTy, NumElts: NumSrcElts);
3781 Value *Load = Builder.CreateAlignedLoad(Ty: VT, Ptr: CI->getArgOperand(i: 0), Align: Align(1));
3782 if (NumSrcElts == 2)
3783 Rep = Builder.CreateShuffleVector(V: Load, Mask: ArrayRef<int>{0, 1, 0, 1});
3784 else
3785 Rep = Builder.CreateShuffleVector(V: Load,
3786 Mask: ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3787 } else if (Name.starts_with(Prefix: "avx512.mask.shuf.i") ||
3788 Name.starts_with(Prefix: "avx512.mask.shuf.f")) {
3789 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3790 Type *VT = CI->getType();
3791 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3792 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3793 unsigned ControlBitsMask = NumLanes - 1;
3794 unsigned NumControlBits = NumLanes / 2;
3795 SmallVector<int, 8> ShuffleMask(0);
3796
3797 for (unsigned l = 0; l != NumLanes; ++l) {
3798 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3799 // We actually need the other source.
3800 if (l >= NumLanes / 2)
3801 LaneMask += NumLanes;
3802 for (unsigned i = 0; i != NumElementsInLane; ++i)
3803 ShuffleMask.push_back(Elt: LaneMask * NumElementsInLane + i);
3804 }
3805 Rep = Builder.CreateShuffleVector(V1: CI->getArgOperand(i: 0),
3806 V2: CI->getArgOperand(i: 1), Mask: ShuffleMask);
3807 Rep =
3808 emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: CI->getArgOperand(i: 3));
3809 } else if (Name.starts_with(Prefix: "avx512.mask.broadcastf") ||
3810 Name.starts_with(Prefix: "avx512.mask.broadcasti")) {
3811 unsigned NumSrcElts = cast<FixedVectorType>(Val: CI->getArgOperand(i: 0)->getType())
3812 ->getNumElements();
3813 unsigned NumDstElts =
3814 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3815
3816 SmallVector<int, 8> ShuffleMask(NumDstElts);
3817 for (unsigned i = 0; i != NumDstElts; ++i)
3818 ShuffleMask[i] = i % NumSrcElts;
3819
3820 Rep = Builder.CreateShuffleVector(V1: CI->getArgOperand(i: 0),
3821 V2: CI->getArgOperand(i: 0), Mask: ShuffleMask);
3822 Rep =
3823 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3824 } else if (Name.starts_with(Prefix: "avx2.pbroadcast") ||
3825 Name.starts_with(Prefix: "avx2.vbroadcast") ||
3826 Name.starts_with(Prefix: "avx512.pbroadcast") ||
3827 Name.starts_with(Prefix: "avx512.mask.broadcast.s")) {
3828 // Replace vp?broadcasts with a vector shuffle.
3829 Value *Op = CI->getArgOperand(i: 0);
3830 ElementCount EC = cast<VectorType>(Val: CI->getType())->getElementCount();
3831 Type *MaskTy = VectorType::get(ElementType: Type::getInt32Ty(C), EC);
3832 SmallVector<int, 8> M;
3833 ShuffleVectorInst::getShuffleMask(Mask: Constant::getNullValue(Ty: MaskTy), Result&: M);
3834 Rep = Builder.CreateShuffleVector(V: Op, Mask: M);
3835
3836 if (CI->arg_size() == 3)
3837 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3838 Op1: CI->getArgOperand(i: 1));
3839 } else if (Name.starts_with(Prefix: "sse2.padds.") ||
3840 Name.starts_with(Prefix: "avx2.padds.") ||
3841 Name.starts_with(Prefix: "avx512.padds.") ||
3842 Name.starts_with(Prefix: "avx512.mask.padds.")) {
3843 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::sadd_sat);
3844 } else if (Name.starts_with(Prefix: "sse2.psubs.") ||
3845 Name.starts_with(Prefix: "avx2.psubs.") ||
3846 Name.starts_with(Prefix: "avx512.psubs.") ||
3847 Name.starts_with(Prefix: "avx512.mask.psubs.")) {
3848 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::ssub_sat);
3849 } else if (Name.starts_with(Prefix: "sse2.paddus.") ||
3850 Name.starts_with(Prefix: "avx2.paddus.") ||
3851 Name.starts_with(Prefix: "avx512.mask.paddus.")) {
3852 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::uadd_sat);
3853 } else if (Name.starts_with(Prefix: "sse2.psubus.") ||
3854 Name.starts_with(Prefix: "avx2.psubus.") ||
3855 Name.starts_with(Prefix: "avx512.mask.psubus.")) {
3856 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::usub_sat);
3857 } else if (Name.starts_with(Prefix: "avx512.mask.palignr.")) {
3858 Rep = upgradeX86ALIGNIntrinsics(Builder, Op0: CI->getArgOperand(i: 0),
3859 Op1: CI->getArgOperand(i: 1), Shift: CI->getArgOperand(i: 2),
3860 Passthru: CI->getArgOperand(i: 3), Mask: CI->getArgOperand(i: 4),
3861 IsVALIGN: false);
3862 } else if (Name.starts_with(Prefix: "avx512.mask.valign.")) {
3863 Rep = upgradeX86ALIGNIntrinsics(
3864 Builder, Op0: CI->getArgOperand(i: 0), Op1: CI->getArgOperand(i: 1),
3865 Shift: CI->getArgOperand(i: 2), Passthru: CI->getArgOperand(i: 3), Mask: CI->getArgOperand(i: 4), IsVALIGN: true);
3866 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
3867 // 128/256-bit shift left specified in bits.
3868 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
3869 Rep = upgradeX86PSLLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0),
3870 Shift: Shift / 8); // Shift is in bits.
3871 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
3872 // 128/256-bit shift right specified in bits.
3873 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
3874 Rep = upgradeX86PSRLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0),
3875 Shift: Shift / 8); // Shift is in bits.
3876 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
3877 Name == "avx512.psll.dq.512") {
3878 // 128/256/512-bit shift left specified in bytes.
3879 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
3880 Rep = upgradeX86PSLLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0), Shift);
3881 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
3882 Name == "avx512.psrl.dq.512") {
3883 // 128/256/512-bit shift right specified in bytes.
3884 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
3885 Rep = upgradeX86PSRLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0), Shift);
3886 } else if (Name == "sse41.pblendw" || Name.starts_with(Prefix: "sse41.blendp") ||
3887 Name.starts_with(Prefix: "avx.blend.p") || Name == "avx2.pblendw" ||
3888 Name.starts_with(Prefix: "avx2.pblendd.")) {
3889 Value *Op0 = CI->getArgOperand(i: 0);
3890 Value *Op1 = CI->getArgOperand(i: 1);
3891 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3892 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
3893 unsigned NumElts = VecTy->getNumElements();
3894
3895 SmallVector<int, 16> Idxs(NumElts);
3896 for (unsigned i = 0; i != NumElts; ++i)
3897 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
3898
3899 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
3900 } else if (Name.starts_with(Prefix: "avx.vinsertf128.") ||
3901 Name == "avx2.vinserti128" ||
3902 Name.starts_with(Prefix: "avx512.mask.insert")) {
3903 Value *Op0 = CI->getArgOperand(i: 0);
3904 Value *Op1 = CI->getArgOperand(i: 1);
3905 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3906 unsigned DstNumElts =
3907 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3908 unsigned SrcNumElts =
3909 cast<FixedVectorType>(Val: Op1->getType())->getNumElements();
3910 unsigned Scale = DstNumElts / SrcNumElts;
3911
3912 // Mask off the high bits of the immediate value; hardware ignores those.
3913 Imm = Imm % Scale;
3914
3915 // Extend the second operand into a vector the size of the destination.
3916 SmallVector<int, 8> Idxs(DstNumElts);
3917 for (unsigned i = 0; i != SrcNumElts; ++i)
3918 Idxs[i] = i;
3919 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
3920 Idxs[i] = SrcNumElts;
3921 Rep = Builder.CreateShuffleVector(V: Op1, Mask: Idxs);
3922
3923 // Insert the second operand into the first operand.
3924
3925 // Note that there is no guarantee that instruction lowering will actually
3926 // produce a vinsertf128 instruction for the created shuffles. In
3927 // particular, the 0 immediate case involves no lane changes, so it can
3928 // be handled as a blend.
3929
3930 // Example of shuffle mask for 32-bit elements:
3931 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
3932 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
3933
3934 // First fill with identify mask.
3935 for (unsigned i = 0; i != DstNumElts; ++i)
3936 Idxs[i] = i;
3937 // Then replace the elements where we need to insert.
3938 for (unsigned i = 0; i != SrcNumElts; ++i)
3939 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3940 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Rep, Mask: Idxs);
3941
3942 // If the intrinsic has a mask operand, handle that.
3943 if (CI->arg_size() == 5)
3944 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep,
3945 Op1: CI->getArgOperand(i: 3));
3946 } else if (Name.starts_with(Prefix: "avx.vextractf128.") ||
3947 Name == "avx2.vextracti128" ||
3948 Name.starts_with(Prefix: "avx512.mask.vextract")) {
3949 Value *Op0 = CI->getArgOperand(i: 0);
3950 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
3951 unsigned DstNumElts =
3952 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3953 unsigned SrcNumElts =
3954 cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
3955 unsigned Scale = SrcNumElts / DstNumElts;
3956
3957 // Mask off the high bits of the immediate value; hardware ignores those.
3958 Imm = Imm % Scale;
3959
3960 // Get indexes for the subvector of the input vector.
3961 SmallVector<int, 8> Idxs(DstNumElts);
3962 for (unsigned i = 0; i != DstNumElts; ++i) {
3963 Idxs[i] = i + (Imm * DstNumElts);
3964 }
3965 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
3966
3967 // If the intrinsic has a mask operand, handle that.
3968 if (CI->arg_size() == 4)
3969 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
3970 Op1: CI->getArgOperand(i: 2));
3971 } else if (Name.starts_with(Prefix: "avx512.mask.perm.df.") ||
3972 Name.starts_with(Prefix: "avx512.mask.perm.di.")) {
3973 Value *Op0 = CI->getArgOperand(i: 0);
3974 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
3975 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
3976 unsigned NumElts = VecTy->getNumElements();
3977
3978 SmallVector<int, 8> Idxs(NumElts);
3979 for (unsigned i = 0; i != NumElts; ++i)
3980 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3981
3982 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
3983
3984 if (CI->arg_size() == 4)
3985 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
3986 Op1: CI->getArgOperand(i: 2));
3987 } else if (Name.starts_with(Prefix: "avx.vperm2f128.") || Name == "avx2.vperm2i128") {
3988 // The immediate permute control byte looks like this:
3989 // [1:0] - select 128 bits from sources for low half of destination
3990 // [2] - ignore
3991 // [3] - zero low half of destination
3992 // [5:4] - select 128 bits from sources for high half of destination
3993 // [6] - ignore
3994 // [7] - zero high half of destination
3995
3996 uint8_t Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3997
3998 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3999 unsigned HalfSize = NumElts / 2;
4000 SmallVector<int, 8> ShuffleMask(NumElts);
4001
4002 // Determine which operand(s) are actually in use for this instruction.
4003 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(i: 1) : CI->getArgOperand(i: 0);
4004 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(i: 1) : CI->getArgOperand(i: 0);
4005
4006 // If needed, replace operands based on zero mask.
4007 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(Ty: CI->getType()) : V0;
4008 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(Ty: CI->getType()) : V1;
4009
4010 // Permute low half of result.
4011 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4012 for (unsigned i = 0; i < HalfSize; ++i)
4013 ShuffleMask[i] = StartIndex + i;
4014
4015 // Permute high half of result.
4016 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4017 for (unsigned i = 0; i < HalfSize; ++i)
4018 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4019
4020 Rep = Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
4021
4022 } else if (Name.starts_with(Prefix: "avx.vpermil.") || Name == "sse2.pshuf.d" ||
4023 Name.starts_with(Prefix: "avx512.mask.vpermil.p") ||
4024 Name.starts_with(Prefix: "avx512.mask.pshuf.d.")) {
4025 Value *Op0 = CI->getArgOperand(i: 0);
4026 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4027 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4028 unsigned NumElts = VecTy->getNumElements();
4029 // Calculate the size of each index in the immediate.
4030 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4031 unsigned IdxMask = ((1 << IdxSize) - 1);
4032
4033 SmallVector<int, 8> Idxs(NumElts);
4034 // Lookup the bits for this element, wrapping around the immediate every
4035 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4036 // to offset by the first index of each group.
4037 for (unsigned i = 0; i != NumElts; ++i)
4038 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4039
4040 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4041
4042 if (CI->arg_size() == 4)
4043 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4044 Op1: CI->getArgOperand(i: 2));
4045 } else if (Name == "sse2.pshufl.w" ||
4046 Name.starts_with(Prefix: "avx512.mask.pshufl.w.")) {
4047 Value *Op0 = CI->getArgOperand(i: 0);
4048 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4049 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4050
4051 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4052 reportFatalUsageErrorWithCI(reason: "Intrinsic has invalid signature", CI);
4053
4054 SmallVector<int, 16> Idxs(NumElts);
4055 for (unsigned l = 0; l != NumElts; l += 8) {
4056 for (unsigned i = 0; i != 4; ++i)
4057 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4058 for (unsigned i = 4; i != 8; ++i)
4059 Idxs[i + l] = i + l;
4060 }
4061
4062 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4063
4064 if (CI->arg_size() == 4)
4065 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4066 Op1: CI->getArgOperand(i: 2));
4067 } else if (Name == "sse2.pshufh.w" ||
4068 Name.starts_with(Prefix: "avx512.mask.pshufh.w.")) {
4069 Value *Op0 = CI->getArgOperand(i: 0);
4070 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4071 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4072
4073 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4074 reportFatalUsageErrorWithCI(reason: "Intrinsic has invalid signature", CI);
4075
4076 SmallVector<int, 16> Idxs(NumElts);
4077 for (unsigned l = 0; l != NumElts; l += 8) {
4078 for (unsigned i = 0; i != 4; ++i)
4079 Idxs[i + l] = i + l;
4080 for (unsigned i = 0; i != 4; ++i)
4081 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4082 }
4083
4084 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4085
4086 if (CI->arg_size() == 4)
4087 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4088 Op1: CI->getArgOperand(i: 2));
4089 } else if (Name.starts_with(Prefix: "avx512.mask.shuf.p")) {
4090 Value *Op0 = CI->getArgOperand(i: 0);
4091 Value *Op1 = CI->getArgOperand(i: 1);
4092 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4093 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4094
4095 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4096 unsigned HalfLaneElts = NumLaneElts / 2;
4097
4098 SmallVector<int, 16> Idxs(NumElts);
4099 for (unsigned i = 0; i != NumElts; ++i) {
4100 // Base index is the starting element of the lane.
4101 Idxs[i] = i - (i % NumLaneElts);
4102 // If we are half way through the lane switch to the other source.
4103 if ((i % NumLaneElts) >= HalfLaneElts)
4104 Idxs[i] += NumElts;
4105 // Now select the specific element. By adding HalfLaneElts bits from
4106 // the immediate. Wrapping around the immediate every 8-bits.
4107 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4108 }
4109
4110 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4111
4112 Rep =
4113 emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: CI->getArgOperand(i: 3));
4114 } else if (Name.starts_with(Prefix: "avx512.mask.movddup") ||
4115 Name.starts_with(Prefix: "avx512.mask.movshdup") ||
4116 Name.starts_with(Prefix: "avx512.mask.movsldup")) {
4117 Value *Op0 = CI->getArgOperand(i: 0);
4118 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4119 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4120
4121 unsigned Offset = 0;
4122 if (Name.starts_with(Prefix: "avx512.mask.movshdup."))
4123 Offset = 1;
4124
4125 SmallVector<int, 16> Idxs(NumElts);
4126 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4127 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4128 Idxs[i + l + 0] = i + l + Offset;
4129 Idxs[i + l + 1] = i + l + Offset;
4130 }
4131
4132 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4133
4134 Rep =
4135 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4136 } else if (Name.starts_with(Prefix: "avx512.mask.punpckl") ||
4137 Name.starts_with(Prefix: "avx512.mask.unpckl.")) {
4138 Value *Op0 = CI->getArgOperand(i: 0);
4139 Value *Op1 = CI->getArgOperand(i: 1);
4140 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4141 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4142
4143 SmallVector<int, 64> Idxs(NumElts);
4144 for (int l = 0; l != NumElts; l += NumLaneElts)
4145 for (int i = 0; i != NumLaneElts; ++i)
4146 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4147
4148 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4149
4150 Rep =
4151 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4152 } else if (Name.starts_with(Prefix: "avx512.mask.punpckh") ||
4153 Name.starts_with(Prefix: "avx512.mask.unpckh.")) {
4154 Value *Op0 = CI->getArgOperand(i: 0);
4155 Value *Op1 = CI->getArgOperand(i: 1);
4156 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4157 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4158
4159 SmallVector<int, 64> Idxs(NumElts);
4160 for (int l = 0; l != NumElts; l += NumLaneElts)
4161 for (int i = 0; i != NumLaneElts; ++i)
4162 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4163
4164 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4165
4166 Rep =
4167 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4168 } else if (Name.starts_with(Prefix: "avx512.mask.and.") ||
4169 Name.starts_with(Prefix: "avx512.mask.pand.")) {
4170 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4171 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4172 Rep = Builder.CreateAnd(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4173 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4174 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4175 Rep =
4176 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4177 } else if (Name.starts_with(Prefix: "avx512.mask.andn.") ||
4178 Name.starts_with(Prefix: "avx512.mask.pandn.")) {
4179 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4180 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4181 Rep = Builder.CreateNot(V: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy));
4182 Rep = Builder.CreateAnd(LHS: Rep,
4183 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4184 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4185 Rep =
4186 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4187 } else if (Name.starts_with(Prefix: "avx512.mask.or.") ||
4188 Name.starts_with(Prefix: "avx512.mask.por.")) {
4189 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4190 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4191 Rep = Builder.CreateOr(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4192 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4193 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4194 Rep =
4195 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4196 } else if (Name.starts_with(Prefix: "avx512.mask.xor.") ||
4197 Name.starts_with(Prefix: "avx512.mask.pxor.")) {
4198 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4199 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4200 Rep = Builder.CreateXor(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4201 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4202 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4203 Rep =
4204 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4205 } else if (Name.starts_with(Prefix: "avx512.mask.padd.")) {
4206 Rep = Builder.CreateAdd(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4207 Rep =
4208 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4209 } else if (Name.starts_with(Prefix: "avx512.mask.psub.")) {
4210 Rep = Builder.CreateSub(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4211 Rep =
4212 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4213 } else if (Name.starts_with(Prefix: "avx512.mask.pmull.")) {
4214 Rep = Builder.CreateMul(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4215 Rep =
4216 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4217 } else if (Name.starts_with(Prefix: "avx512.mask.add.p")) {
4218 if (Name.ends_with(Suffix: ".512")) {
4219 Intrinsic::ID IID;
4220 if (Name[17] == 's')
4221 IID = Intrinsic::x86_avx512_add_ps_512;
4222 else
4223 IID = Intrinsic::x86_avx512_add_pd_512;
4224
4225 Rep = Builder.CreateIntrinsic(
4226 ID: IID,
4227 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4228 } else {
4229 Rep = Builder.CreateFAdd(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4230 }
4231 Rep =
4232 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4233 } else if (Name.starts_with(Prefix: "avx512.mask.div.p")) {
4234 if (Name.ends_with(Suffix: ".512")) {
4235 Intrinsic::ID IID;
4236 if (Name[17] == 's')
4237 IID = Intrinsic::x86_avx512_div_ps_512;
4238 else
4239 IID = Intrinsic::x86_avx512_div_pd_512;
4240
4241 Rep = Builder.CreateIntrinsic(
4242 ID: IID,
4243 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4244 } else {
4245 Rep = Builder.CreateFDiv(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4246 }
4247 Rep =
4248 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4249 } else if (Name.starts_with(Prefix: "avx512.mask.mul.p")) {
4250 if (Name.ends_with(Suffix: ".512")) {
4251 Intrinsic::ID IID;
4252 if (Name[17] == 's')
4253 IID = Intrinsic::x86_avx512_mul_ps_512;
4254 else
4255 IID = Intrinsic::x86_avx512_mul_pd_512;
4256
4257 Rep = Builder.CreateIntrinsic(
4258 ID: IID,
4259 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4260 } else {
4261 Rep = Builder.CreateFMul(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4262 }
4263 Rep =
4264 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4265 } else if (Name.starts_with(Prefix: "avx512.mask.sub.p")) {
4266 if (Name.ends_with(Suffix: ".512")) {
4267 Intrinsic::ID IID;
4268 if (Name[17] == 's')
4269 IID = Intrinsic::x86_avx512_sub_ps_512;
4270 else
4271 IID = Intrinsic::x86_avx512_sub_pd_512;
4272
4273 Rep = Builder.CreateIntrinsic(
4274 ID: IID,
4275 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4276 } else {
4277 Rep = Builder.CreateFSub(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4278 }
4279 Rep =
4280 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4281 } else if ((Name.starts_with(Prefix: "avx512.mask.max.p") ||
4282 Name.starts_with(Prefix: "avx512.mask.min.p")) &&
4283 Name.drop_front(N: 18) == ".512") {
4284 bool IsDouble = Name[17] == 'd';
4285 bool IsMin = Name[13] == 'i';
4286 static const Intrinsic::ID MinMaxTbl[2][2] = {
4287 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4288 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4289 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4290
4291 Rep = Builder.CreateIntrinsic(
4292 ID: IID,
4293 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4294 Rep =
4295 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4296 } else if (Name.starts_with(Prefix: "avx512.mask.lzcnt.")) {
4297 Rep =
4298 Builder.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: CI->getType(),
4299 Args: {CI->getArgOperand(i: 0), Builder.getInt1(V: false)});
4300 Rep =
4301 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4302 } else if (Name.starts_with(Prefix: "avx512.mask.psll")) {
4303 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4304 bool IsVariable = Name[16] == 'v';
4305 char Size = Name[16] == '.' ? Name[17]
4306 : Name[17] == '.' ? Name[18]
4307 : Name[18] == '.' ? Name[19]
4308 : Name[20];
4309
4310 Intrinsic::ID IID;
4311 if (IsVariable && Name[17] != '.') {
4312 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4313 IID = Intrinsic::x86_avx2_psllv_q;
4314 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4315 IID = Intrinsic::x86_avx2_psllv_q_256;
4316 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4317 IID = Intrinsic::x86_avx2_psllv_d;
4318 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4319 IID = Intrinsic::x86_avx2_psllv_d_256;
4320 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4321 IID = Intrinsic::x86_avx512_psllv_w_128;
4322 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4323 IID = Intrinsic::x86_avx512_psllv_w_256;
4324 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4325 IID = Intrinsic::x86_avx512_psllv_w_512;
4326 else
4327 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4328 } else if (Name.ends_with(Suffix: ".128")) {
4329 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4330 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4331 : Intrinsic::x86_sse2_psll_d;
4332 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4333 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4334 : Intrinsic::x86_sse2_psll_q;
4335 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4336 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4337 : Intrinsic::x86_sse2_psll_w;
4338 else
4339 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4340 } else if (Name.ends_with(Suffix: ".256")) {
4341 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4342 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4343 : Intrinsic::x86_avx2_psll_d;
4344 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4345 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4346 : Intrinsic::x86_avx2_psll_q;
4347 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4348 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4349 : Intrinsic::x86_avx2_psll_w;
4350 else
4351 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4352 } else {
4353 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4354 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4355 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4356 : Intrinsic::x86_avx512_psll_d_512;
4357 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4358 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4359 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4360 : Intrinsic::x86_avx512_psll_q_512;
4361 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4362 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4363 : Intrinsic::x86_avx512_psll_w_512;
4364 else
4365 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4366 }
4367
4368 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4369 } else if (Name.starts_with(Prefix: "avx512.mask.psrl")) {
4370 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4371 bool IsVariable = Name[16] == 'v';
4372 char Size = Name[16] == '.' ? Name[17]
4373 : Name[17] == '.' ? Name[18]
4374 : Name[18] == '.' ? Name[19]
4375 : Name[20];
4376
4377 Intrinsic::ID IID;
4378 if (IsVariable && Name[17] != '.') {
4379 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4380 IID = Intrinsic::x86_avx2_psrlv_q;
4381 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4382 IID = Intrinsic::x86_avx2_psrlv_q_256;
4383 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4384 IID = Intrinsic::x86_avx2_psrlv_d;
4385 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4386 IID = Intrinsic::x86_avx2_psrlv_d_256;
4387 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4388 IID = Intrinsic::x86_avx512_psrlv_w_128;
4389 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4390 IID = Intrinsic::x86_avx512_psrlv_w_256;
4391 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4392 IID = Intrinsic::x86_avx512_psrlv_w_512;
4393 else
4394 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4395 } else if (Name.ends_with(Suffix: ".128")) {
4396 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4397 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4398 : Intrinsic::x86_sse2_psrl_d;
4399 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4400 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4401 : Intrinsic::x86_sse2_psrl_q;
4402 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4403 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4404 : Intrinsic::x86_sse2_psrl_w;
4405 else
4406 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4407 } else if (Name.ends_with(Suffix: ".256")) {
4408 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4409 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4410 : Intrinsic::x86_avx2_psrl_d;
4411 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4412 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4413 : Intrinsic::x86_avx2_psrl_q;
4414 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4415 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4416 : Intrinsic::x86_avx2_psrl_w;
4417 else
4418 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4419 } else {
4420 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4421 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4422 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4423 : Intrinsic::x86_avx512_psrl_d_512;
4424 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4425 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4426 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4427 : Intrinsic::x86_avx512_psrl_q_512;
4428 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4429 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4430 : Intrinsic::x86_avx512_psrl_w_512;
4431 else
4432 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4433 }
4434
4435 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4436 } else if (Name.starts_with(Prefix: "avx512.mask.psra")) {
4437 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4438 bool IsVariable = Name[16] == 'v';
4439 char Size = Name[16] == '.' ? Name[17]
4440 : Name[17] == '.' ? Name[18]
4441 : Name[18] == '.' ? Name[19]
4442 : Name[20];
4443
4444 Intrinsic::ID IID;
4445 if (IsVariable && Name[17] != '.') {
4446 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4447 IID = Intrinsic::x86_avx2_psrav_d;
4448 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4449 IID = Intrinsic::x86_avx2_psrav_d_256;
4450 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4451 IID = Intrinsic::x86_avx512_psrav_w_128;
4452 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4453 IID = Intrinsic::x86_avx512_psrav_w_256;
4454 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4455 IID = Intrinsic::x86_avx512_psrav_w_512;
4456 else
4457 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4458 } else if (Name.ends_with(Suffix: ".128")) {
4459 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4460 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4461 : Intrinsic::x86_sse2_psra_d;
4462 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4463 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4464 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4465 : Intrinsic::x86_avx512_psra_q_128;
4466 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4467 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4468 : Intrinsic::x86_sse2_psra_w;
4469 else
4470 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4471 } else if (Name.ends_with(Suffix: ".256")) {
4472 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4473 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4474 : Intrinsic::x86_avx2_psra_d;
4475 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4476 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4477 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4478 : Intrinsic::x86_avx512_psra_q_256;
4479 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4480 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4481 : Intrinsic::x86_avx2_psra_w;
4482 else
4483 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4484 } else {
4485 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4486 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4487 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4488 : Intrinsic::x86_avx512_psra_d_512;
4489 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4490 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4491 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4492 : Intrinsic::x86_avx512_psra_q_512;
4493 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4494 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4495 : Intrinsic::x86_avx512_psra_w_512;
4496 else
4497 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4498 }
4499
4500 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4501 } else if (Name.starts_with(Prefix: "avx512.mask.move.s")) {
4502 Rep = upgradeMaskedMove(Builder, CI&: *CI);
4503 } else if (Name.starts_with(Prefix: "avx512.cvtmask2")) {
4504 Rep = upgradeMaskToInt(Builder, CI&: *CI);
4505 } else if (Name.ends_with(Suffix: ".movntdqa")) {
4506 MDNode *Node = MDNode::get(
4507 Context&: C, MDs: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
4508
4509 LoadInst *LI = Builder.CreateAlignedLoad(
4510 Ty: CI->getType(), Ptr: CI->getArgOperand(i: 0),
4511 Align: Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4512 LI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
4513 Rep = LI;
4514 } else if (Name.starts_with(Prefix: "fma.vfmadd.") ||
4515 Name.starts_with(Prefix: "fma.vfmsub.") ||
4516 Name.starts_with(Prefix: "fma.vfnmadd.") ||
4517 Name.starts_with(Prefix: "fma.vfnmsub.")) {
4518 bool NegMul = Name[6] == 'n';
4519 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4520 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4521
4522 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4523 CI->getArgOperand(i: 2)};
4524
4525 if (IsScalar) {
4526 Ops[0] = Builder.CreateExtractElement(Vec: Ops[0], Idx: (uint64_t)0);
4527 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: (uint64_t)0);
4528 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: (uint64_t)0);
4529 }
4530
4531 if (NegMul && !IsScalar)
4532 Ops[0] = Builder.CreateFNeg(V: Ops[0]);
4533 if (NegMul && IsScalar)
4534 Ops[1] = Builder.CreateFNeg(V: Ops[1]);
4535 if (NegAcc)
4536 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4537
4538 Rep = Builder.CreateIntrinsic(ID: Intrinsic::fma, OverloadTypes: Ops[0]->getType(), Args: Ops);
4539
4540 if (IsScalar)
4541 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
4542 } else if (Name.starts_with(Prefix: "fma4.vfmadd.s")) {
4543 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4544 CI->getArgOperand(i: 2)};
4545
4546 Ops[0] = Builder.CreateExtractElement(Vec: Ops[0], Idx: (uint64_t)0);
4547 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: (uint64_t)0);
4548 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: (uint64_t)0);
4549
4550 Rep = Builder.CreateIntrinsic(ID: Intrinsic::fma, OverloadTypes: Ops[0]->getType(), Args: Ops);
4551
4552 Rep = Builder.CreateInsertElement(Vec: Constant::getNullValue(Ty: CI->getType()),
4553 NewElt: Rep, Idx: (uint64_t)0);
4554 } else if (Name.starts_with(Prefix: "avx512.mask.vfmadd.s") ||
4555 Name.starts_with(Prefix: "avx512.maskz.vfmadd.s") ||
4556 Name.starts_with(Prefix: "avx512.mask3.vfmadd.s") ||
4557 Name.starts_with(Prefix: "avx512.mask3.vfmsub.s") ||
4558 Name.starts_with(Prefix: "avx512.mask3.vfnmsub.s")) {
4559 bool IsMask3 = Name[11] == '3';
4560 bool IsMaskZ = Name[11] == 'z';
4561 // Drop the "avx512.mask." to make it easier.
4562 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4563 bool NegMul = Name[2] == 'n';
4564 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4565
4566 Value *A = CI->getArgOperand(i: 0);
4567 Value *B = CI->getArgOperand(i: 1);
4568 Value *C = CI->getArgOperand(i: 2);
4569
4570 if (NegMul && (IsMask3 || IsMaskZ))
4571 A = Builder.CreateFNeg(V: A);
4572 if (NegMul && !(IsMask3 || IsMaskZ))
4573 B = Builder.CreateFNeg(V: B);
4574 if (NegAcc)
4575 C = Builder.CreateFNeg(V: C);
4576
4577 A = Builder.CreateExtractElement(Vec: A, Idx: (uint64_t)0);
4578 B = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0);
4579 C = Builder.CreateExtractElement(Vec: C, Idx: (uint64_t)0);
4580
4581 if (!isa<ConstantInt>(Val: CI->getArgOperand(i: 4)) ||
4582 cast<ConstantInt>(Val: CI->getArgOperand(i: 4))->getZExtValue() != 4) {
4583 Value *Ops[] = {A, B, C, CI->getArgOperand(i: 4)};
4584
4585 Intrinsic::ID IID;
4586 if (Name.back() == 'd')
4587 IID = Intrinsic::x86_avx512_vfmadd_f64;
4588 else
4589 IID = Intrinsic::x86_avx512_vfmadd_f32;
4590 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4591 } else {
4592 Rep = Builder.CreateFMA(Factor1: A, Factor2: B, Summand: C);
4593 }
4594
4595 Value *PassThru = IsMaskZ ? Constant::getNullValue(Ty: Rep->getType())
4596 : IsMask3 ? C
4597 : A;
4598
4599 // For Mask3 with NegAcc, we need to create a new extractelement that
4600 // avoids the negation above.
4601 if (NegAcc && IsMask3)
4602 PassThru =
4603 Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 2), Idx: (uint64_t)0);
4604
4605 Rep = emitX86ScalarSelect(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4606 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: IsMask3 ? 2 : 0), NewElt: Rep,
4607 Idx: (uint64_t)0);
4608 } else if (Name.starts_with(Prefix: "avx512.mask.vfmadd.p") ||
4609 Name.starts_with(Prefix: "avx512.mask.vfnmadd.p") ||
4610 Name.starts_with(Prefix: "avx512.mask.vfnmsub.p") ||
4611 Name.starts_with(Prefix: "avx512.mask3.vfmadd.p") ||
4612 Name.starts_with(Prefix: "avx512.mask3.vfmsub.p") ||
4613 Name.starts_with(Prefix: "avx512.mask3.vfnmsub.p") ||
4614 Name.starts_with(Prefix: "avx512.maskz.vfmadd.p")) {
4615 bool IsMask3 = Name[11] == '3';
4616 bool IsMaskZ = Name[11] == 'z';
4617 // Drop the "avx512.mask." to make it easier.
4618 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4619 bool NegMul = Name[2] == 'n';
4620 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4621
4622 Value *A = CI->getArgOperand(i: 0);
4623 Value *B = CI->getArgOperand(i: 1);
4624 Value *C = CI->getArgOperand(i: 2);
4625
4626 if (NegMul && (IsMask3 || IsMaskZ))
4627 A = Builder.CreateFNeg(V: A);
4628 if (NegMul && !(IsMask3 || IsMaskZ))
4629 B = Builder.CreateFNeg(V: B);
4630 if (NegAcc)
4631 C = Builder.CreateFNeg(V: C);
4632
4633 if (CI->arg_size() == 5 &&
4634 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 4)) ||
4635 cast<ConstantInt>(Val: CI->getArgOperand(i: 4))->getZExtValue() != 4)) {
4636 Intrinsic::ID IID;
4637 // Check the character before ".512" in string.
4638 if (Name[Name.size() - 5] == 's')
4639 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4640 else
4641 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4642
4643 Rep = Builder.CreateIntrinsic(ID: IID, Args: {A, B, C, CI->getArgOperand(i: 4)});
4644 } else {
4645 Rep = Builder.CreateFMA(Factor1: A, Factor2: B, Summand: C);
4646 }
4647
4648 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(Ty: CI->getType())
4649 : IsMask3 ? CI->getArgOperand(i: 2)
4650 : CI->getArgOperand(i: 0);
4651
4652 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4653 } else if (Name.starts_with(Prefix: "fma.vfmsubadd.p")) {
4654 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4655 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4656 Intrinsic::ID IID;
4657 if (VecWidth == 128 && EltWidth == 32)
4658 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4659 else if (VecWidth == 256 && EltWidth == 32)
4660 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4661 else if (VecWidth == 128 && EltWidth == 64)
4662 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4663 else if (VecWidth == 256 && EltWidth == 64)
4664 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4665 else
4666 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4667
4668 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4669 CI->getArgOperand(i: 2)};
4670 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4671 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4672 } else if (Name.starts_with(Prefix: "avx512.mask.vfmaddsub.p") ||
4673 Name.starts_with(Prefix: "avx512.mask3.vfmaddsub.p") ||
4674 Name.starts_with(Prefix: "avx512.maskz.vfmaddsub.p") ||
4675 Name.starts_with(Prefix: "avx512.mask3.vfmsubadd.p")) {
4676 bool IsMask3 = Name[11] == '3';
4677 bool IsMaskZ = Name[11] == 'z';
4678 // Drop the "avx512.mask." to make it easier.
4679 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4680 bool IsSubAdd = Name[3] == 's';
4681 if (CI->arg_size() == 5) {
4682 Intrinsic::ID IID;
4683 // Check the character before ".512" in string.
4684 if (Name[Name.size() - 5] == 's')
4685 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4686 else
4687 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4688
4689 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4690 CI->getArgOperand(i: 2), CI->getArgOperand(i: 4)};
4691 if (IsSubAdd)
4692 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4693
4694 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4695 } else {
4696 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4697
4698 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4699 CI->getArgOperand(i: 2)};
4700
4701 Function *FMA = Intrinsic::getOrInsertDeclaration(
4702 M: CI->getModule(), id: Intrinsic::fma, OverloadTys: Ops[0]->getType());
4703 Value *Odd = Builder.CreateCall(Callee: FMA, Args: Ops);
4704 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4705 Value *Even = Builder.CreateCall(Callee: FMA, Args: Ops);
4706
4707 if (IsSubAdd)
4708 std::swap(a&: Even, b&: Odd);
4709
4710 SmallVector<int, 32> Idxs(NumElts);
4711 for (int i = 0; i != NumElts; ++i)
4712 Idxs[i] = i + (i % 2) * NumElts;
4713
4714 Rep = Builder.CreateShuffleVector(V1: Even, V2: Odd, Mask: Idxs);
4715 }
4716
4717 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(Ty: CI->getType())
4718 : IsMask3 ? CI->getArgOperand(i: 2)
4719 : CI->getArgOperand(i: 0);
4720
4721 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4722 } else if (Name.starts_with(Prefix: "avx512.mask.pternlog.") ||
4723 Name.starts_with(Prefix: "avx512.maskz.pternlog.")) {
4724 bool ZeroMask = Name[11] == 'z';
4725 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4726 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4727 Intrinsic::ID IID;
4728 if (VecWidth == 128 && EltWidth == 32)
4729 IID = Intrinsic::x86_avx512_pternlog_d_128;
4730 else if (VecWidth == 256 && EltWidth == 32)
4731 IID = Intrinsic::x86_avx512_pternlog_d_256;
4732 else if (VecWidth == 512 && EltWidth == 32)
4733 IID = Intrinsic::x86_avx512_pternlog_d_512;
4734 else if (VecWidth == 128 && EltWidth == 64)
4735 IID = Intrinsic::x86_avx512_pternlog_q_128;
4736 else if (VecWidth == 256 && EltWidth == 64)
4737 IID = Intrinsic::x86_avx512_pternlog_q_256;
4738 else if (VecWidth == 512 && EltWidth == 64)
4739 IID = Intrinsic::x86_avx512_pternlog_q_512;
4740 else
4741 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4742
4743 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4744 CI->getArgOperand(i: 2), CI->getArgOperand(i: 3)};
4745 Rep = Builder.CreateIntrinsic(ID: IID, Args);
4746 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
4747 : CI->getArgOperand(i: 0);
4748 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: PassThru);
4749 } else if (Name.starts_with(Prefix: "avx512.mask.vpmadd52") ||
4750 Name.starts_with(Prefix: "avx512.maskz.vpmadd52")) {
4751 bool ZeroMask = Name[11] == 'z';
4752 bool High = Name[20] == 'h' || Name[21] == 'h';
4753 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4754 Intrinsic::ID IID;
4755 if (VecWidth == 128 && !High)
4756 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4757 else if (VecWidth == 256 && !High)
4758 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4759 else if (VecWidth == 512 && !High)
4760 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4761 else if (VecWidth == 128 && High)
4762 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4763 else if (VecWidth == 256 && High)
4764 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4765 else if (VecWidth == 512 && High)
4766 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4767 else
4768 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4769
4770 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4771 CI->getArgOperand(i: 2)};
4772 Rep = Builder.CreateIntrinsic(ID: IID, Args);
4773 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
4774 : CI->getArgOperand(i: 0);
4775 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4776 } else if (Name.starts_with(Prefix: "avx512.mask.vpermi2var.") ||
4777 Name.starts_with(Prefix: "avx512.mask.vpermt2var.") ||
4778 Name.starts_with(Prefix: "avx512.maskz.vpermt2var.")) {
4779 bool ZeroMask = Name[11] == 'z';
4780 bool IndexForm = Name[17] == 'i';
4781 Rep = upgradeX86VPERMT2Intrinsics(Builder, CI&: *CI, ZeroMask, IndexForm);
4782 } else if (Name.starts_with(Prefix: "avx512.mask.vpdpbusd.") ||
4783 Name.starts_with(Prefix: "avx512.maskz.vpdpbusd.") ||
4784 Name.starts_with(Prefix: "avx512.mask.vpdpbusds.") ||
4785 Name.starts_with(Prefix: "avx512.maskz.vpdpbusds.")) {
4786 bool ZeroMask = Name[11] == 'z';
4787 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4788 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4789 Intrinsic::ID IID;
4790 if (VecWidth == 128 && !IsSaturating)
4791 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4792 else if (VecWidth == 256 && !IsSaturating)
4793 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4794 else if (VecWidth == 512 && !IsSaturating)
4795 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4796 else if (VecWidth == 128 && IsSaturating)
4797 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4798 else if (VecWidth == 256 && IsSaturating)
4799 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4800 else if (VecWidth == 512 && IsSaturating)
4801 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4802 else
4803 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4804
4805 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4806 CI->getArgOperand(i: 2)};
4807
4808 // Input arguments types were incorrectly set to vectors of i32 before but
4809 // they should be vectors of i8. Insert bit cast when encountering the old
4810 // types
4811 if (Args[1]->getType()->isVectorTy() &&
4812 cast<VectorType>(Val: Args[1]->getType())
4813 ->getElementType()
4814 ->isIntegerTy(BitWidth: 32) &&
4815 Args[2]->getType()->isVectorTy() &&
4816 cast<VectorType>(Val: Args[2]->getType())
4817 ->getElementType()
4818 ->isIntegerTy(BitWidth: 32)) {
4819 Type *NewArgType = nullptr;
4820 if (VecWidth == 128)
4821 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 16, Scalable: false);
4822 else if (VecWidth == 256)
4823 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 32, Scalable: false);
4824 else if (VecWidth == 512)
4825 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 64, Scalable: false);
4826 else
4827 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected vector bit width",
4828 CI);
4829
4830 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
4831 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
4832 }
4833
4834 Rep = Builder.CreateIntrinsic(ID: IID, Args);
4835 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
4836 : CI->getArgOperand(i: 0);
4837 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4838 } else if (Name.starts_with(Prefix: "avx512.mask.vpdpwssd.") ||
4839 Name.starts_with(Prefix: "avx512.maskz.vpdpwssd.") ||
4840 Name.starts_with(Prefix: "avx512.mask.vpdpwssds.") ||
4841 Name.starts_with(Prefix: "avx512.maskz.vpdpwssds.")) {
4842 bool ZeroMask = Name[11] == 'z';
4843 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4844 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4845 Intrinsic::ID IID;
4846 if (VecWidth == 128 && !IsSaturating)
4847 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4848 else if (VecWidth == 256 && !IsSaturating)
4849 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4850 else if (VecWidth == 512 && !IsSaturating)
4851 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4852 else if (VecWidth == 128 && IsSaturating)
4853 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4854 else if (VecWidth == 256 && IsSaturating)
4855 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4856 else if (VecWidth == 512 && IsSaturating)
4857 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4858 else
4859 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4860
4861 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4862 CI->getArgOperand(i: 2)};
4863
4864 // Input arguments types were incorrectly set to vectors of i32 before but
4865 // they should be vectors of i16. Insert bit cast when encountering the old
4866 // types
4867 if (Args[1]->getType()->isVectorTy() &&
4868 cast<VectorType>(Val: Args[1]->getType())
4869 ->getElementType()
4870 ->isIntegerTy(BitWidth: 32) &&
4871 Args[2]->getType()->isVectorTy() &&
4872 cast<VectorType>(Val: Args[2]->getType())
4873 ->getElementType()
4874 ->isIntegerTy(BitWidth: 32)) {
4875 Type *NewArgType = nullptr;
4876 if (VecWidth == 128)
4877 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 8, Scalable: false);
4878 else if (VecWidth == 256)
4879 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 16, Scalable: false);
4880 else if (VecWidth == 512)
4881 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 32, Scalable: false);
4882 else
4883 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected vector bit width",
4884 CI);
4885
4886 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
4887 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
4888 }
4889
4890 Rep = Builder.CreateIntrinsic(ID: IID, Args);
4891 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
4892 : CI->getArgOperand(i: 0);
4893 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4894 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
4895 Name == "addcarry.u32" || Name == "addcarry.u64" ||
4896 Name == "subborrow.u32" || Name == "subborrow.u64") {
4897 Intrinsic::ID IID;
4898 if (Name[0] == 'a' && Name.back() == '2')
4899 IID = Intrinsic::x86_addcarry_32;
4900 else if (Name[0] == 'a' && Name.back() == '4')
4901 IID = Intrinsic::x86_addcarry_64;
4902 else if (Name[0] == 's' && Name.back() == '2')
4903 IID = Intrinsic::x86_subborrow_32;
4904 else if (Name[0] == 's' && Name.back() == '4')
4905 IID = Intrinsic::x86_subborrow_64;
4906 else
4907 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4908
4909 // Make a call with 3 operands.
4910 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4911 CI->getArgOperand(i: 2)};
4912 Value *NewCall = Builder.CreateIntrinsic(ID: IID, Args);
4913
4914 // Extract the second result and store it.
4915 Value *Data = Builder.CreateExtractValue(Agg: NewCall, Idxs: 1);
4916 Builder.CreateAlignedStore(Val: Data, Ptr: CI->getArgOperand(i: 3), Align: Align(1));
4917 // Replace the original call result with the first result of the new call.
4918 Value *CF = Builder.CreateExtractValue(Agg: NewCall, Idxs: 0);
4919
4920 CI->replaceAllUsesWith(V: CF);
4921 Rep = nullptr;
4922 } else if (Name.starts_with(Prefix: "avx512.mask.") &&
4923 upgradeAVX512MaskToSelect(Name, Builder, CI&: *CI, Rep)) {
4924 // Rep will be updated by the call in the condition.
4925 } else if (Name.starts_with(Prefix: "bmi.pdep.")) {
4926 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::pdep);
4927 } else if (Name.starts_with(Prefix: "bmi.pext.")) {
4928 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::pext);
4929 } else
4930 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4931
4932 return Rep;
4933}
4934
4935static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
4936 Function *F, IRBuilder<> &Builder) {
4937 if (Name.starts_with(Prefix: "neon.bfcvt")) {
4938 if (Name.starts_with(Prefix: "neon.bfcvtn2")) {
4939 SmallVector<int, 32> LoMask(4);
4940 std::iota(first: LoMask.begin(), last: LoMask.end(), value: 0);
4941 SmallVector<int, 32> ConcatMask(8);
4942 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
4943 Value *Inactive = Builder.CreateShuffleVector(V: CI->getOperand(i_nocapture: 0), Mask: LoMask);
4944 Value *Trunc =
4945 Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 1), DestTy: Inactive->getType());
4946 return Builder.CreateShuffleVector(V1: Inactive, V2: Trunc, Mask: ConcatMask);
4947 } else if (Name.starts_with(Prefix: "neon.bfcvtn")) {
4948 SmallVector<int, 32> ConcatMask(8);
4949 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
4950 Type *V4BF16 =
4951 FixedVectorType::get(ElementType: Type::getBFloatTy(C&: F->getContext()), NumElts: 4);
4952 Value *Trunc = Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 0), DestTy: V4BF16);
4953 dbgs() << "Trunc: " << *Trunc << "\n";
4954 return Builder.CreateShuffleVector(
4955 V1: Trunc, V2: ConstantAggregateZero::get(Ty: V4BF16), Mask: ConcatMask);
4956 } else {
4957 return Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 0),
4958 DestTy: Type::getBFloatTy(C&: F->getContext()));
4959 }
4960 } else if (Name.starts_with(Prefix: "sve.fcvt")) {
4961 Intrinsic::ID NewID =
4962 StringSwitch<Intrinsic::ID>(Name)
4963 .Case(S: "sve.fcvt.bf16f32", Value: Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
4964 .Case(S: "sve.fcvtnt.bf16f32",
4965 Value: Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
4966 .Default(Value: Intrinsic::not_intrinsic);
4967 if (NewID == Intrinsic::not_intrinsic)
4968 llvm_unreachable("Unhandled Intrinsic!");
4969
4970 SmallVector<Value *, 3> Args(CI->args());
4971
4972 // The original intrinsics incorrectly used a predicate based on the
4973 // smallest element type rather than the largest.
4974 Type *BadPredTy = ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
4975 Type *GoodPredTy = ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
4976
4977 if (Args[1]->getType() != BadPredTy)
4978 llvm_unreachable("Unexpected predicate type!");
4979
4980 Args[1] = Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_convert_to_svbool,
4981 OverloadTypes: BadPredTy, Args: Args[1]);
4982 Args[1] = Builder.CreateIntrinsic(
4983 ID: Intrinsic::aarch64_sve_convert_from_svbool, OverloadTypes: GoodPredTy, Args: Args[1]);
4984
4985 return Builder.CreateIntrinsic(ID: NewID, Args, /*FMFSource=*/nullptr,
4986 Name: CI->getName());
4987 }
4988
4989 if (Name == "neon.vcvtfp2hf")
4990 return Builder.CreateBitCast(
4991 V: Builder.CreateFPTrunc(
4992 V: CI->getOperand(i_nocapture: 0),
4993 DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C&: F->getContext()), NumElts: 4)),
4994 DestTy: FixedVectorType::get(ElementType: Type::getInt16Ty(C&: F->getContext()), NumElts: 4));
4995 if (Name == "neon.vcvthf2fp")
4996 return Builder.CreateFPExt(
4997 V: Builder.CreateBitCast(
4998 V: CI->getOperand(i_nocapture: 0),
4999 DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C&: F->getContext()), NumElts: 4)),
5000 DestTy: FixedVectorType::get(ElementType: Type::getFloatTy(C&: F->getContext()), NumElts: 4));
5001
5002 llvm_unreachable("Unhandled Intrinsic!");
5003}
5004
5005static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5006 IRBuilder<> &Builder) {
5007 if (Name == "mve.vctp64.old") {
5008 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5009 // correct type.
5010 Value *VCTP = Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_vctp64, OverloadTypes: {},
5011 Args: CI->getArgOperand(i: 0),
5012 /*FMFSource=*/nullptr, Name: CI->getName());
5013 Value *C1 = Builder.CreateIntrinsic(
5014 ID: Intrinsic::arm_mve_pred_v2i,
5015 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 2, Scalable: false)}, Args: VCTP);
5016 return Builder.CreateIntrinsic(
5017 ID: Intrinsic::arm_mve_pred_i2v,
5018 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 4, Scalable: false)}, Args: C1);
5019 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5020 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5021 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5022 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5023 Name ==
5024 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5025 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5026 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5027 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5028 Name ==
5029 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5030 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5031 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5032 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5033 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5034 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5035 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5036 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5037 std::vector<Type *> Tys;
5038 unsigned ID = CI->getIntrinsicID();
5039 Type *V2I1Ty = FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts: 2);
5040 switch (ID) {
5041 case Intrinsic::arm_mve_mull_int_predicated:
5042 case Intrinsic::arm_mve_vqdmull_predicated:
5043 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5044 Tys = {CI->getType(), CI->getOperand(i_nocapture: 0)->getType(), V2I1Ty};
5045 break;
5046 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5047 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5048 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5049 Tys = {CI->getOperand(i_nocapture: 0)->getType(), CI->getOperand(i_nocapture: 0)->getType(),
5050 V2I1Ty};
5051 break;
5052 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5053 Tys = {CI->getType(), CI->getOperand(i_nocapture: 0)->getType(),
5054 CI->getOperand(i_nocapture: 1)->getType(), V2I1Ty};
5055 break;
5056 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5057 Tys = {CI->getOperand(i_nocapture: 0)->getType(), CI->getOperand(i_nocapture: 1)->getType(),
5058 CI->getOperand(i_nocapture: 2)->getType(), V2I1Ty};
5059 break;
5060 case Intrinsic::arm_cde_vcx1q_predicated:
5061 case Intrinsic::arm_cde_vcx1qa_predicated:
5062 case Intrinsic::arm_cde_vcx2q_predicated:
5063 case Intrinsic::arm_cde_vcx2qa_predicated:
5064 case Intrinsic::arm_cde_vcx3q_predicated:
5065 case Intrinsic::arm_cde_vcx3qa_predicated:
5066 Tys = {CI->getOperand(i_nocapture: 1)->getType(), V2I1Ty};
5067 break;
5068 default:
5069 llvm_unreachable("Unhandled Intrinsic!");
5070 }
5071
5072 std::vector<Value *> Ops;
5073 for (Value *Op : CI->args()) {
5074 Type *Ty = Op->getType();
5075 if (Ty->getScalarSizeInBits() == 1) {
5076 Value *C1 = Builder.CreateIntrinsic(
5077 ID: Intrinsic::arm_mve_pred_v2i,
5078 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 4, Scalable: false)}, Args: Op);
5079 Op = Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_pred_i2v, OverloadTypes: {V2I1Ty}, Args: C1);
5080 }
5081 Ops.push_back(x: Op);
5082 }
5083
5084 return Builder.CreateIntrinsic(ID, OverloadTypes: Tys, Args: Ops, /*FMFSource=*/nullptr,
5085 Name: CI->getName());
5086 }
5087 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5088}
5089
5090// These are expected to have the arguments:
5091// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5092//
5093// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5094//
5095static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5096 Function *F, IRBuilder<> &Builder) {
5097 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5098 // for compatibility.
5099 auto UpgradeLegacyWMMAIUIntrinsicCall =
5100 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5101 ArrayRef<Type *> OverloadTys) -> Value * {
5102 // Prepare arguments, append clamp=0 for compatibility
5103 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5104 Args.push_back(Elt: Builder.getFalse());
5105
5106 // Insert the declaration for the right overload types
5107 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5108 M: F->getParent(), id: F->getIntrinsicID(), OverloadTys);
5109
5110 // Copy operand bundles if any
5111 SmallVector<OperandBundleDef, 1> Bundles;
5112 CI->getOperandBundlesAsDefs(Defs&: Bundles);
5113
5114 // Create the new call and copy calling properties
5115 auto *NewCall = cast<CallInst>(Val: Builder.CreateCall(Callee: NewDecl, Args, OpBundles: Bundles));
5116 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5117 NewCall->setCallingConv(CI->getCallingConv());
5118 NewCall->setAttributes(CI->getAttributes());
5119 NewCall->copyMetadata(SrcInst: *CI);
5120 return NewCall;
5121 };
5122
5123 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5124 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5125 "intrinsic should have 7 arguments");
5126 Type *T1 = CI->getArgOperand(i: 4)->getType();
5127 Type *T2 = CI->getArgOperand(i: 1)->getType();
5128 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5129 }
5130 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5131 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5132 "intrinsic should have 8 arguments");
5133 Type *T1 = CI->getArgOperand(i: 4)->getType();
5134 Type *T2 = CI->getArgOperand(i: 1)->getType();
5135 Type *T3 = CI->getArgOperand(i: 3)->getType();
5136 Type *T4 = CI->getArgOperand(i: 5)->getType();
5137 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5138 }
5139
5140 switch (F->getIntrinsicID()) {
5141 default:
5142 break;
5143 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5144 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5145 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5146 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5147 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5148 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5149 // Drop src0 and src1 modifiers.
5150 const Value *Op0 = CI->getArgOperand(i: 0);
5151 const Value *Op2 = CI->getArgOperand(i: 2);
5152 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5153 const ConstantInt *ModA = dyn_cast<ConstantInt>(Val: Op0);
5154 const ConstantInt *ModB = dyn_cast<ConstantInt>(Val: Op2);
5155 if (!ModA->isZero() || !ModB->isZero())
5156 reportFatalUsageError(reason: Name + " matrix A and B modifiers shall be zero");
5157
5158 SmallVector<Value *, 8> Args{CI->getArgOperand(i: 1), CI->getArgOperand(i: 3)};
5159 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5160 Args.push_back(Elt: CI->getArgOperand(i: I));
5161
5162 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5163 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5164 Overloads.push_back(Elt: Args[3]->getType());
5165 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5166 M: F->getParent(), id: F->getIntrinsicID(), OverloadTys: Overloads);
5167
5168 SmallVector<OperandBundleDef, 1> Bundles;
5169 CI->getOperandBundlesAsDefs(Defs&: Bundles);
5170
5171 auto *NewCall = cast<CallInst>(Val: Builder.CreateCall(Callee: NewDecl, Args, OpBundles: Bundles));
5172 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5173 NewCall->setCallingConv(CI->getCallingConv());
5174 NewCall->setAttributes(CI->getAttributes());
5175 NewCall->copyMetadata(SrcInst: *CI);
5176 NewCall->takeName(V: CI);
5177 return NewCall;
5178 }
5179 }
5180
5181 if (Name.starts_with(Prefix: "fcmp.") || Name.starts_with(Prefix: "icmp.")) {
5182 Value *LHS = CI->getArgOperand(i: 0);
5183 Value *RHS = CI->getArgOperand(i: 1);
5184 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5185 cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue());
5186 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5187 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5188 RetTy: CI->getType(), ID: Intrinsic::amdgcn_ballot, Args: Cmp);
5189 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5190 NewCall->setCallingConv(CI->getCallingConv());
5191 NewCall->copyMetadata(SrcInst: *CI);
5192 NewCall->takeName(V: CI);
5193 return NewCall;
5194 }
5195
5196 AtomicRMWInst::BinOp RMWOp =
5197 StringSwitch<AtomicRMWInst::BinOp>(Name)
5198 .StartsWith(S: "ds.fadd", Value: AtomicRMWInst::FAdd)
5199 .StartsWith(S: "ds.fmin", Value: AtomicRMWInst::FMin)
5200 .StartsWith(S: "ds.fmax", Value: AtomicRMWInst::FMax)
5201 .StartsWith(S: "atomic.inc.", Value: AtomicRMWInst::UIncWrap)
5202 .StartsWith(S: "atomic.dec.", Value: AtomicRMWInst::UDecWrap)
5203 .StartsWith(S: "global.atomic.fadd", Value: AtomicRMWInst::FAdd)
5204 .StartsWith(S: "flat.atomic.fadd", Value: AtomicRMWInst::FAdd)
5205 .StartsWith(S: "global.atomic.fmin", Value: AtomicRMWInst::FMin)
5206 .StartsWith(S: "flat.atomic.fmin", Value: AtomicRMWInst::FMin)
5207 .StartsWith(S: "global.atomic.fmax", Value: AtomicRMWInst::FMax)
5208 .StartsWith(S: "flat.atomic.fmax", Value: AtomicRMWInst::FMax)
5209 .StartsWith(S: "atomic.cond.sub", Value: AtomicRMWInst::USubCond)
5210 .StartsWith(S: "atomic.csub", Value: AtomicRMWInst::USubSat);
5211
5212 unsigned NumOperands = CI->getNumOperands();
5213 if (NumOperands < 3) // Malformed bitcode.
5214 return nullptr;
5215
5216 Value *Ptr = CI->getArgOperand(i: 0);
5217 PointerType *PtrTy = dyn_cast<PointerType>(Val: Ptr->getType());
5218 if (!PtrTy) // Malformed.
5219 return nullptr;
5220
5221 Value *Val = CI->getArgOperand(i: 1);
5222 if (Val->getType() != CI->getType()) // Malformed.
5223 return nullptr;
5224
5225 ConstantInt *OrderArg = nullptr;
5226 bool IsVolatile = false;
5227
5228 // These should have 5 arguments (plus the callee). A separate version of the
5229 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5230 if (NumOperands > 3)
5231 OrderArg = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
5232
5233 // Ignore scope argument at 3
5234
5235 if (NumOperands > 5) {
5236 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 4));
5237 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5238 }
5239
5240 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5241 if (OrderArg && isValidAtomicOrdering(I: OrderArg->getZExtValue()))
5242 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5243 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5244 Order = AtomicOrdering::SequentiallyConsistent;
5245
5246 LLVMContext &Ctx = F->getContext();
5247
5248 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5249 Type *RetTy = CI->getType();
5250 if (VectorType *VT = dyn_cast<VectorType>(Val: RetTy)) {
5251 if (VT->getElementType()->isIntegerTy(BitWidth: 16)) {
5252 VectorType *AsBF16 =
5253 VectorType::get(ElementType: Type::getBFloatTy(C&: Ctx), EC: VT->getElementCount());
5254 Val = Builder.CreateBitCast(V: Val, DestTy: AsBF16);
5255 }
5256 }
5257
5258 // The scope argument never really worked correctly. Use agent as the most
5259 // conservative option which should still always produce the instruction.
5260 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID(SSN: "agent");
5261 AtomicRMWInst *RMW =
5262 Builder.CreateAtomicRMW(Op: RMWOp, Ptr, Val, Align: std::nullopt, Ordering: Order, SSID);
5263
5264 unsigned AddrSpace = PtrTy->getAddressSpace();
5265 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5266 MDNode *EmptyMD = MDNode::get(Context&: F->getContext(), MDs: {});
5267 RMW->setMetadata(Kind: "amdgpu.no.fine.grained.memory", Node: EmptyMD);
5268 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5269 RMW->setMetadata(Kind: "amdgpu.ignore.denormal.mode", Node: EmptyMD);
5270 }
5271
5272 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5273 MDBuilder MDB(F->getContext());
5274 MDNode *RangeNotPrivate =
5275 MDB.createRange(Lo: APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5276 Hi: APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5277 RMW->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: RangeNotPrivate);
5278 }
5279
5280 if (IsVolatile)
5281 RMW->setVolatile(true);
5282
5283 return Builder.CreateBitCast(V: RMW, DestTy: RetTy);
5284}
5285
5286/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5287/// plain MDNode, as it's the verifier's job to check these are the correct
5288/// types later.
5289static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5290 if (Op < CI->arg_size()) {
5291 if (MetadataAsValue *MAV =
5292 dyn_cast<MetadataAsValue>(Val: CI->getArgOperand(i: Op))) {
5293 Metadata *MD = MAV->getMetadata();
5294 return dyn_cast_if_present<MDNode>(Val: MD);
5295 }
5296 }
5297 return nullptr;
5298}
5299
5300/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5301static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5302 if (Op < CI->arg_size())
5303 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(Val: CI->getArgOperand(i: Op)))
5304 return MAV->getMetadata();
5305 return nullptr;
5306}
5307
5308/// Convert debug intrinsic calls to non-instruction debug records.
5309/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5310/// \p CI - The debug intrinsic call.
5311static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5312 DbgRecord *DR = nullptr;
5313 if (Name == "label") {
5314 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(Label: unwrapMAVOp(CI, Op: 0));
5315 } else if (Name == "assign") {
5316 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5317 Type: DbgVariableRecord::LocationType::Assign, Val: unwrapMAVMetadataOp(CI, Op: 0),
5318 Variable: unwrapMAVOp(CI, Op: 1), Expression: unwrapMAVOp(CI, Op: 2), AssignID: unwrapMAVOp(CI, Op: 3),
5319 Address: unwrapMAVMetadataOp(CI, Op: 4),
5320 /*The address is a Value ref, it will be stored as a Metadata */
5321 AddressExpression: unwrapMAVOp(CI, Op: 5));
5322 } else if (Name == "declare") {
5323 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5324 Type: DbgVariableRecord::LocationType::Declare, Val: unwrapMAVMetadataOp(CI, Op: 0),
5325 Variable: unwrapMAVOp(CI, Op: 1), Expression: unwrapMAVOp(CI, Op: 2), AssignID: nullptr, Address: nullptr, AddressExpression: nullptr);
5326 } else if (Name == "addr") {
5327 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5328 MDNode *ExprNode = unwrapMAVOp(CI, Op: 2);
5329 // Don't try to add something to the expression if it's not an expression.
5330 // Instead, allow the verifier to fail later.
5331 if (DIExpression *Expr = dyn_cast<DIExpression>(Val: ExprNode)) {
5332 ExprNode = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
5333 }
5334 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5335 Type: DbgVariableRecord::LocationType::Value, Val: unwrapMAVMetadataOp(CI, Op: 0),
5336 Variable: unwrapMAVOp(CI, Op: 1), Expression: ExprNode, AssignID: nullptr, Address: nullptr, AddressExpression: nullptr);
5337 } else if (Name == "value") {
5338 // An old version of dbg.value had an extra offset argument.
5339 unsigned VarOp = 1;
5340 unsigned ExprOp = 2;
5341 if (CI->arg_size() == 4) {
5342 auto *Offset = dyn_cast_or_null<Constant>(Val: CI->getArgOperand(i: 1));
5343 // Nonzero offset dbg.values get dropped without a replacement.
5344 if (!Offset || !Offset->isNullValue())
5345 return;
5346 VarOp = 2;
5347 ExprOp = 3;
5348 }
5349 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5350 Type: DbgVariableRecord::LocationType::Value, Val: unwrapMAVMetadataOp(CI, Op: 0),
5351 Variable: unwrapMAVOp(CI, Op: VarOp), Expression: unwrapMAVOp(CI, Op: ExprOp), AssignID: nullptr, Address: nullptr,
5352 AddressExpression: nullptr);
5353 }
5354 DR->setDebugLoc(CI->getDebugLoc());
5355 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5356 CI->getParent()->insertDbgRecordBefore(DR, Here: CI->getIterator());
5357}
5358
5359static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5360 auto *Offset = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
5361 if (!Offset)
5362 reportFatalUsageError(reason: "Invalid llvm.vector.splice offset argument");
5363 int64_t OffsetVal = Offset->getSExtValue();
5364 return Builder.CreateIntrinsic(ID: OffsetVal >= 0
5365 ? Intrinsic::vector_splice_left
5366 : Intrinsic::vector_splice_right,
5367 OverloadTypes: CI->getType(),
5368 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5369 Builder.getInt32(C: std::abs(i: OffsetVal))});
5370}
5371
5372static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5373 Function *F, IRBuilder<> &Builder) {
5374 if (Name.starts_with(Prefix: "to.fp16")) {
5375 Value *Cast =
5376 Builder.CreateFPTrunc(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
5377 return Builder.CreateBitCast(V: Cast, DestTy: CI->getType());
5378 }
5379
5380 if (Name.starts_with(Prefix: "from.fp16")) {
5381 Value *Cast =
5382 Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
5383 return Builder.CreateFPExt(V: Cast, DestTy: CI->getType());
5384 }
5385
5386 return nullptr;
5387}
5388
5389static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5390 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
5391 if (!MD || !isa<MDString>(Val: MD))
5392 return ICmpInst::BAD_ICMP_PREDICATE;
5393 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
5394 .Case(S: "eq", Value: ICmpInst::ICMP_EQ)
5395 .Case(S: "ne", Value: ICmpInst::ICMP_NE)
5396 .Case(S: "ugt", Value: ICmpInst::ICMP_UGT)
5397 .Case(S: "uge", Value: ICmpInst::ICMP_UGE)
5398 .Case(S: "ult", Value: ICmpInst::ICMP_ULT)
5399 .Case(S: "ule", Value: ICmpInst::ICMP_ULE)
5400 .Case(S: "sgt", Value: ICmpInst::ICMP_SGT)
5401 .Case(S: "sge", Value: ICmpInst::ICMP_SGE)
5402 .Case(S: "slt", Value: ICmpInst::ICMP_SLT)
5403 .Case(S: "sle", Value: ICmpInst::ICMP_SLE)
5404 .Default(Value: ICmpInst::BAD_ICMP_PREDICATE);
5405}
5406
5407static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5408 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
5409 if (!MD || !isa<MDString>(Val: MD))
5410 return FCmpInst::BAD_FCMP_PREDICATE;
5411 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
5412 .Case(S: "oeq", Value: FCmpInst::FCMP_OEQ)
5413 .Case(S: "ogt", Value: FCmpInst::FCMP_OGT)
5414 .Case(S: "oge", Value: FCmpInst::FCMP_OGE)
5415 .Case(S: "olt", Value: FCmpInst::FCMP_OLT)
5416 .Case(S: "ole", Value: FCmpInst::FCMP_OLE)
5417 .Case(S: "one", Value: FCmpInst::FCMP_ONE)
5418 .Case(S: "ord", Value: FCmpInst::FCMP_ORD)
5419 .Case(S: "uno", Value: FCmpInst::FCMP_UNO)
5420 .Case(S: "ueq", Value: FCmpInst::FCMP_UEQ)
5421 .Case(S: "ugt", Value: FCmpInst::FCMP_UGT)
5422 .Case(S: "uge", Value: FCmpInst::FCMP_UGE)
5423 .Case(S: "ult", Value: FCmpInst::FCMP_ULT)
5424 .Case(S: "ule", Value: FCmpInst::FCMP_ULE)
5425 .Case(S: "une", Value: FCmpInst::FCMP_UNE)
5426 .Default(Value: FCmpInst::BAD_FCMP_PREDICATE);
5427}
5428
5429static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5430 IRBuilder<> &Builder) {
5431 Value *Rep;
5432 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5433 if (Opcode && Instruction::isUnaryOp(Opcode))
5434 Rep =
5435 Builder.CreateUnOp(Opc: (Instruction::UnaryOps)Opcode, V: CI->getArgOperand(i: 0));
5436 else if (Opcode && Instruction::isBinaryOp(Opcode))
5437 Rep = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Opcode,
5438 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5439 else if (Opcode && Instruction::isCast(Opcode))
5440 Rep = Builder.CreateCast(Op: (Instruction::CastOps)Opcode, V: CI->getArgOperand(i: 0),
5441 DestTy: CI->getType());
5442 else if (Opcode == Instruction::ICmp)
5443 Rep = Builder.CreateICmp(P: getVPIntPredicateFromMD(Op: CI->getArgOperand(i: 2)),
5444 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5445 else if (Opcode == Instruction::FCmp)
5446 Rep = Builder.CreateFCmp(P: getVPFPPredicateFromMD(Op: CI->getArgOperand(i: 2)),
5447 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5448 else if (Opcode == Instruction::Select)
5449 Rep = Builder.CreateSelect(C: CI->getArgOperand(i: 0), True: CI->getArgOperand(i: 1),
5450 False: CI->getArgOperand(i: 2));
5451 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5452 SmallVector<Value *, 2> Args(drop_end(RangeOrContainer: CI->args(), N: 2));
5453 Rep = Builder.CreateIntrinsic(RetTy: CI->getType(), ID: IntrinsicID, Args, FMFSource: {});
5454 } else
5455 llvm_unreachable("Unexpected vp intrinsic");
5456 Rep->takeName(V: CI);
5457 return Rep;
5458}
5459
5460static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5461 IRBuilder<> &Builder) {
5462 Intrinsic::ID IID = NewFn->getIntrinsicID();
5463
5464 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5465 if (Defaults.empty())
5466 return false;
5467
5468 unsigned OldArgCount = CI->arg_size();
5469 unsigned NewArgCount = NewFn->arg_size();
5470
5471 // If the caller already supplied all arguments (or more), nothing to do.
5472 // This mirrors C++ semantics: an explicitly-passed value is never overridden.
5473 if (OldArgCount >= NewArgCount)
5474 return false;
5475
5476 // Start with the existing arguments from the old call.
5477 SmallVector<Value *, 8> NewArgs(CI->args());
5478
5479 // Defaults are a contiguous trailing block, so checking the first missing
5480 // argument is enough.
5481 if (OldArgCount < FirstDefault)
5482 return false;
5483
5484 // Fill in each missing trailing argument from the table.
5485 FunctionType *NewFT = NewFn->getFunctionType();
5486 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5487 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5488 "missing argument outside the default range");
5489 Type *ParamTy = NewFT->getParamType(i: Idx);
5490
5491 // Only integer types are supported (i1, i8, i16, i32, i64).
5492 if (!ParamTy->isIntegerTy())
5493 return false;
5494 NewArgs.push_back(Elt: ConstantInt::get(Ty: ParamTy, V: Defaults[Idx - FirstDefault]));
5495 }
5496
5497 // Preserve operand bundles by creating the call with them.
5498 SmallVector<OperandBundleDef, 1> OpBundles;
5499 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
5500 CallInst *NewCall = Builder.CreateCall(Callee: NewFn, Args: NewArgs, OpBundles);
5501
5502 NewCall->takeName(V: CI);
5503 NewCall->setCallingConv(CI->getCallingConv());
5504 NewCall->copyMetadata(SrcInst: *CI);
5505 if (auto *OldCI = dyn_cast<CallInst>(Val: CI))
5506 NewCall->setTailCallKind(OldCI->getTailCallKind());
5507
5508 CI->replaceAllUsesWith(V: NewCall);
5509 CI->eraseFromParent();
5510 return true;
5511}
5512
5513/// Upgrade a call to an old intrinsic. All argument and return casting must be
5514/// provided to seamlessly integrate with existing context.
5515void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5516 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5517 // checks the callee's function type matches. It's likely we need to handle
5518 // type changes here.
5519 Function *F = dyn_cast<Function>(Val: CI->getCalledOperand());
5520 if (!F)
5521 return;
5522
5523 LLVMContext &C = CI->getContext();
5524 IRBuilder<> Builder(C);
5525 if (isa<FPMathOperator>(Val: CI))
5526 Builder.setFastMathFlags(CI->getFastMathFlags());
5527 Builder.SetInsertPoint(TheBB: CI->getParent(), IP: CI->getIterator());
5528
5529 if (!NewFn) {
5530 // Get the Function's name.
5531 StringRef Name = F->getName();
5532 if (!Name.consume_front(Prefix: "llvm."))
5533 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5534
5535 bool IsX86 = Name.consume_front(Prefix: "x86.");
5536 bool IsNVVM = Name.consume_front(Prefix: "nvvm.");
5537 bool IsAArch64 = Name.consume_front(Prefix: "aarch64.");
5538 bool IsARM = Name.consume_front(Prefix: "arm.");
5539 bool IsAMDGCN = Name.consume_front(Prefix: "amdgcn.");
5540 bool IsDbg = Name.consume_front(Prefix: "dbg.");
5541 bool IsOldSplice =
5542 (Name.consume_front(Prefix: "experimental.vector.splice") ||
5543 Name.consume_front(Prefix: "vector.splice")) &&
5544 !(Name.starts_with(Prefix: ".left") || Name.starts_with(Prefix: ".right"));
5545 Value *Rep = nullptr;
5546
5547 if (!IsX86 && Name == "stackprotectorcheck") {
5548 Rep = nullptr;
5549 } else if (IsNVVM) {
5550 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5551 } else if (IsX86) {
5552 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5553 } else if (IsAArch64) {
5554 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5555 } else if (IsARM) {
5556 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5557 } else if (IsAMDGCN) {
5558 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5559 } else if (IsDbg) {
5560 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5561 } else if (IsOldSplice) {
5562 Rep = upgradeVectorSplice(CI, Builder);
5563 } else if (Name.consume_front(Prefix: "convert.")) {
5564 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5565 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5566 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5567 Rep = nullptr;
5568 } else if (shouldUpgradeVPIntrinsic(Name)) {
5569 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5570 } else {
5571 llvm_unreachable("Unknown function for CallBase upgrade.");
5572 }
5573
5574 if (Rep)
5575 CI->replaceAllUsesWith(V: Rep);
5576 CI->eraseFromParent();
5577 return;
5578 }
5579
5580 const auto &DefaultCase = [&]() -> void {
5581 if (F == NewFn)
5582 return;
5583
5584 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5585 // Handle generic mangling change.
5586 assert(
5587 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5588 "Unknown function for CallBase upgrade and isn't just a name change");
5589 CI->setCalledFunction(NewFn);
5590 return;
5591 }
5592
5593 // This must be an upgrade from a named to a literal struct.
5594 if (auto *OldST = dyn_cast<StructType>(Val: CI->getType())) {
5595 assert(OldST != NewFn->getReturnType() &&
5596 "Return type must have changed");
5597 assert(OldST->getNumElements() ==
5598 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5599 "Must have same number of elements");
5600
5601 SmallVector<Value *> Args(CI->args());
5602 CallInst *NewCI = Builder.CreateCall(Callee: NewFn, Args);
5603 NewCI->setAttributes(CI->getAttributes());
5604 Value *Res = PoisonValue::get(T: OldST);
5605 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5606 Value *Elem = Builder.CreateExtractValue(Agg: NewCI, Idxs: Idx);
5607 Res = Builder.CreateInsertValue(Agg: Res, Val: Elem, Idxs: Idx);
5608 }
5609 CI->replaceAllUsesWith(V: Res);
5610 CI->eraseFromParent();
5611 return;
5612 }
5613
5614 // We're probably about to produce something invalid. Let the verifier catch
5615 // it instead of dying here.
5616 CI->setCalledOperand(
5617 ConstantExpr::getPointerCast(C: NewFn, Ty: CI->getCalledOperand()->getType()));
5618 return;
5619 };
5620 CallInst *NewCall = nullptr;
5621 switch (NewFn->getIntrinsicID()) {
5622 default: {
5623 // Last resort: try the data-driven default-arg upgrade.
5624 // Handles any intrinsic annotated with ImmArg<..., DefaultValue<...>>
5625 // in its .td definition, without needing a dedicated case.
5626 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5627 return;
5628 DefaultCase();
5629 return;
5630 }
5631 case Intrinsic::arm_neon_vst1:
5632 case Intrinsic::arm_neon_vst2:
5633 case Intrinsic::arm_neon_vst3:
5634 case Intrinsic::arm_neon_vst4:
5635 case Intrinsic::arm_neon_vst2lane:
5636 case Intrinsic::arm_neon_vst3lane:
5637 case Intrinsic::arm_neon_vst4lane: {
5638 SmallVector<Value *, 4> Args(CI->args());
5639 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5640 break;
5641 }
5642 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5643 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5644 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5645 LLVMContext &Ctx = F->getParent()->getContext();
5646 SmallVector<Value *, 4> Args(CI->args());
5647 Args[3] = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx),
5648 V: cast<ConstantInt>(Val: Args[3])->getZExtValue());
5649 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5650 break;
5651 }
5652 case Intrinsic::aarch64_sve_ld3_sret:
5653 case Intrinsic::aarch64_sve_ld4_sret:
5654 case Intrinsic::aarch64_sve_ld2_sret: {
5655 // Is this a trivial remangle of the name to support ptr address spaces?
5656 if (isa<StructType>(Val: F->getReturnType())) {
5657 DefaultCase();
5658 return;
5659 }
5660
5661 StringRef Name = F->getName();
5662 Name = Name.substr(Start: 5);
5663 unsigned N = StringSwitch<unsigned>(Name)
5664 .StartsWith(S: "aarch64.sve.ld2", Value: 2)
5665 .StartsWith(S: "aarch64.sve.ld3", Value: 3)
5666 .StartsWith(S: "aarch64.sve.ld4", Value: 4)
5667 .Default(Value: 0);
5668 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5669 unsigned MinElts = RetTy->getMinNumElements() / N;
5670 SmallVector<Value *, 2> Args(CI->args());
5671 Value *NewLdCall = Builder.CreateCall(Callee: NewFn, Args);
5672 Value *Ret = llvm::PoisonValue::get(T: RetTy);
5673 for (unsigned I = 0; I < N; I++) {
5674 Value *SRet = Builder.CreateExtractValue(Agg: NewLdCall, Idxs: I);
5675 Ret = Builder.CreateInsertVector(DstType: RetTy, SrcVec: Ret, SubVec: SRet, Idx: I * MinElts);
5676 }
5677 NewCall = dyn_cast<CallInst>(Val: Ret);
5678 break;
5679 }
5680
5681 case Intrinsic::coro_end_async:
5682 case Intrinsic::coro_end: {
5683 SmallVector<Value *, 3> Args(CI->args());
5684 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5685 Args.push_back(Elt: ConstantTokenNone::get(Context&: CI->getContext()));
5686 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5687
5688 if (!CI->getType()->isVoidTy()) {
5689 if (!CI->use_empty()) {
5690 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5691 M: CI->getModule(), id: Intrinsic::coro_is_in_ramp);
5692 Value *InRamp = Builder.CreateCall(Callee: IsInRamp);
5693 CI->replaceAllUsesWith(V: Builder.CreateNot(V: InRamp));
5694 }
5695 CI->eraseFromParent();
5696 return;
5697 }
5698
5699 break;
5700 }
5701
5702 case Intrinsic::vector_extract: {
5703 StringRef Name = F->getName();
5704 Name = Name.substr(Start: 5); // Strip llvm
5705 if (!Name.starts_with(Prefix: "aarch64.sve.tuple.get")) {
5706 DefaultCase();
5707 return;
5708 }
5709 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5710 unsigned MinElts = RetTy->getMinNumElements();
5711 unsigned I = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
5712 Value *NewIdx = ConstantInt::get(Ty: Type::getInt64Ty(C), V: I * MinElts);
5713 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), NewIdx});
5714 break;
5715 }
5716
5717 case Intrinsic::vector_insert: {
5718 StringRef Name = F->getName();
5719 Name = Name.substr(Start: 5);
5720 if (!Name.starts_with(Prefix: "aarch64.sve.tuple")) {
5721 DefaultCase();
5722 return;
5723 }
5724 if (Name.starts_with(Prefix: "aarch64.sve.tuple.set")) {
5725 unsigned I = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
5726 auto *Ty = cast<ScalableVectorType>(Val: CI->getArgOperand(i: 2)->getType());
5727 Value *NewIdx =
5728 ConstantInt::get(Ty: Type::getInt64Ty(C), V: I * Ty->getMinNumElements());
5729 NewCall = Builder.CreateCall(
5730 Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 2), NewIdx});
5731 break;
5732 }
5733 if (Name.starts_with(Prefix: "aarch64.sve.tuple.create")) {
5734 unsigned N = StringSwitch<unsigned>(Name)
5735 .StartsWith(S: "aarch64.sve.tuple.create2", Value: 2)
5736 .StartsWith(S: "aarch64.sve.tuple.create3", Value: 3)
5737 .StartsWith(S: "aarch64.sve.tuple.create4", Value: 4)
5738 .Default(Value: 0);
5739 assert(N > 1 && "Create is expected to be between 2-4");
5740 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5741 Value *Ret = llvm::PoisonValue::get(T: RetTy);
5742 unsigned MinElts = RetTy->getMinNumElements() / N;
5743 for (unsigned I = 0; I < N; I++) {
5744 Value *V = CI->getArgOperand(i: I);
5745 Ret = Builder.CreateInsertVector(DstType: RetTy, SrcVec: Ret, SubVec: V, Idx: I * MinElts);
5746 }
5747 NewCall = dyn_cast<CallInst>(Val: Ret);
5748 }
5749 break;
5750 }
5751
5752 case Intrinsic::arm_neon_bfdot:
5753 case Intrinsic::arm_neon_bfmmla:
5754 case Intrinsic::arm_neon_bfmlalb:
5755 case Intrinsic::arm_neon_bfmlalt:
5756 case Intrinsic::aarch64_neon_bfdot:
5757 case Intrinsic::aarch64_neon_bfmmla:
5758 case Intrinsic::aarch64_neon_bfmlalb:
5759 case Intrinsic::aarch64_neon_bfmlalt: {
5760 SmallVector<Value *, 3> Args;
5761 assert(CI->arg_size() == 3 &&
5762 "Mismatch between function args and call args");
5763 size_t OperandWidth =
5764 CI->getArgOperand(i: 1)->getType()->getPrimitiveSizeInBits();
5765 assert((OperandWidth == 64 || OperandWidth == 128) &&
5766 "Unexpected operand width");
5767 Type *NewTy = FixedVectorType::get(ElementType: Type::getBFloatTy(C), NumElts: OperandWidth / 16);
5768 auto Iter = CI->args().begin();
5769 Args.push_back(Elt: *Iter++);
5770 Args.push_back(Elt: Builder.CreateBitCast(V: *Iter++, DestTy: NewTy));
5771 Args.push_back(Elt: Builder.CreateBitCast(V: *Iter++, DestTy: NewTy));
5772 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5773 break;
5774 }
5775
5776 case Intrinsic::bitreverse:
5777 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0)});
5778 break;
5779
5780 case Intrinsic::ctlz:
5781 case Intrinsic::cttz: {
5782 if (CI->arg_size() != 1) {
5783 DefaultCase();
5784 return;
5785 }
5786
5787 NewCall =
5788 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), Builder.getFalse()});
5789 break;
5790 }
5791
5792 case Intrinsic::objectsize: {
5793 Value *NullIsUnknownSize =
5794 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(i: 2);
5795 Value *Dynamic =
5796 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(i: 3);
5797 NewCall = Builder.CreateCall(
5798 Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), NullIsUnknownSize, Dynamic});
5799 break;
5800 }
5801
5802 case Intrinsic::ctpop:
5803 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0)});
5804 break;
5805 case Intrinsic::dbg_value: {
5806 StringRef Name = F->getName();
5807 Name = Name.substr(Start: 5); // Strip llvm.
5808 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
5809 if (Name.starts_with(Prefix: "dbg.addr")) {
5810 DIExpression *Expr = cast<DIExpression>(
5811 Val: cast<MetadataAsValue>(Val: CI->getArgOperand(i: 2))->getMetadata());
5812 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
5813 NewCall =
5814 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5815 MetadataAsValue::get(Context&: C, MD: Expr)});
5816 break;
5817 }
5818
5819 // Upgrade from the old version that had an extra offset argument.
5820 assert(CI->arg_size() == 4);
5821 // Drop nonzero offsets instead of attempting to upgrade them.
5822 if (auto *Offset = dyn_cast_or_null<Constant>(Val: CI->getArgOperand(i: 1)))
5823 if (Offset->isNullValue()) {
5824 NewCall = Builder.CreateCall(
5825 Callee: NewFn,
5826 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 2), CI->getArgOperand(i: 3)});
5827 break;
5828 }
5829 CI->eraseFromParent();
5830 return;
5831 }
5832
5833 case Intrinsic::ptr_annotation:
5834 // Upgrade from versions that lacked the annotation attribute argument.
5835 if (CI->arg_size() != 4) {
5836 DefaultCase();
5837 return;
5838 }
5839
5840 // Create a new call with an added null annotation attribute argument.
5841 NewCall = Builder.CreateCall(
5842 Callee: NewFn,
5843 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 2),
5844 CI->getArgOperand(i: 3), ConstantPointerNull::get(T: Builder.getPtrTy())});
5845 NewCall->takeName(V: CI);
5846 CI->replaceAllUsesWith(V: NewCall);
5847 CI->eraseFromParent();
5848 return;
5849
5850 case Intrinsic::var_annotation:
5851 // Upgrade from versions that lacked the annotation attribute argument.
5852 if (CI->arg_size() != 4) {
5853 DefaultCase();
5854 return;
5855 }
5856 // Create a new call with an added null annotation attribute argument.
5857 NewCall = Builder.CreateCall(
5858 Callee: NewFn,
5859 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 2),
5860 CI->getArgOperand(i: 3), ConstantPointerNull::get(T: Builder.getPtrTy())});
5861 NewCall->takeName(V: CI);
5862 CI->replaceAllUsesWith(V: NewCall);
5863 CI->eraseFromParent();
5864 return;
5865
5866 case Intrinsic::riscv_aes32dsi:
5867 case Intrinsic::riscv_aes32dsmi:
5868 case Intrinsic::riscv_aes32esi:
5869 case Intrinsic::riscv_aes32esmi:
5870 case Intrinsic::riscv_sm4ks:
5871 case Intrinsic::riscv_sm4ed: {
5872 // The last argument to these intrinsics used to be i8 and changed to i32.
5873 // The type overload for sm4ks and sm4ed was removed.
5874 Value *Arg2 = CI->getArgOperand(i: 2);
5875 if (Arg2->getType()->isIntegerTy(BitWidth: 32) && !CI->getType()->isIntegerTy(BitWidth: 64))
5876 return;
5877
5878 Value *Arg0 = CI->getArgOperand(i: 0);
5879 Value *Arg1 = CI->getArgOperand(i: 1);
5880 if (CI->getType()->isIntegerTy(BitWidth: 64)) {
5881 Arg0 = Builder.CreateTrunc(V: Arg0, DestTy: Builder.getInt32Ty());
5882 Arg1 = Builder.CreateTrunc(V: Arg1, DestTy: Builder.getInt32Ty());
5883 }
5884
5885 Arg2 = ConstantInt::get(Ty: Type::getInt32Ty(C),
5886 V: cast<ConstantInt>(Val: Arg2)->getZExtValue());
5887
5888 NewCall = Builder.CreateCall(Callee: NewFn, Args: {Arg0, Arg1, Arg2});
5889 Value *Res = NewCall;
5890 if (Res->getType() != CI->getType())
5891 Res = Builder.CreateIntCast(V: NewCall, DestTy: CI->getType(), /*isSigned*/ true);
5892 NewCall->takeName(V: CI);
5893 CI->replaceAllUsesWith(V: Res);
5894 CI->eraseFromParent();
5895 return;
5896 }
5897 case Intrinsic::nvvm_mapa_shared_cluster: {
5898 // Create a new call with the correct address space.
5899 NewCall =
5900 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
5901 Value *Res = NewCall;
5902 Res = Builder.CreateAddrSpaceCast(
5903 V: Res, DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED));
5904 NewCall->takeName(V: CI);
5905 CI->replaceAllUsesWith(V: Res);
5906 CI->eraseFromParent();
5907 return;
5908 }
5909 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
5910 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
5911 // Create a new call with the correct address space.
5912 SmallVector<Value *, 4> Args(CI->args());
5913 Args[0] = Builder.CreateAddrSpaceCast(
5914 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5915
5916 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5917 NewCall->takeName(V: CI);
5918 CI->replaceAllUsesWith(V: NewCall);
5919 CI->eraseFromParent();
5920 return;
5921 }
5922 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d:
5923 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d:
5924 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d:
5925 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d:
5926 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d:
5927 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d:
5928 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d:
5929 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d: {
5930 SmallVector<Value *, 16> Args(CI->args());
5931
5932 // Create AddrSpaceCast to shared_cluster if needed.
5933 // This handles case (1) in shouldUpgradeNVPTXTMAG2SIntrinsics().
5934 unsigned AS = CI->getArgOperand(i: 0)->getType()->getPointerAddressSpace();
5935 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
5936 Args[0] = Builder.CreateAddrSpaceCast(
5937 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5938
5939 // Attach the flag argument for cta_group, with a
5940 // default value of 0. This handles case (2) in
5941 // shouldUpgradeNVPTXTMAG2SIntrinsics().
5942 size_t NumArgs = CI->arg_size();
5943 Value *FlagArg = CI->getArgOperand(i: NumArgs - 3);
5944 if (!FlagArg->getType()->isIntegerTy(BitWidth: 1))
5945 Args.push_back(Elt: ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0));
5946
5947 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5948 NewCall->takeName(V: CI);
5949 CI->replaceAllUsesWith(V: NewCall);
5950 CI->eraseFromParent();
5951 return;
5952 }
5953 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
5954 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
5955 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
5956 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
5957 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
5958 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
5959 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
5960 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
5961 StringRef Name = F->getName();
5962 Name.consume_front(Prefix: "llvm.nvvm.cp.async.bulk.tensor.reduce.");
5963 auto RedOp = getNVPTXTMAReductionOp(Name: Name.split(Separator: '.').first);
5964
5965 SmallVector<Value *, 16> Args(CI->args());
5966 Args.insert(I: Args.end() - 1, Elt: Builder.getInt32(C: *RedOp));
5967 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5968 break;
5969 }
5970 case Intrinsic::nvvm_tcgen05_mma_shared:
5971 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
5972 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
5973 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale:
5974 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale_block32:
5975 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block16:
5976 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block32:
5977 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale:
5978 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale_block32:
5979 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d:
5980 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
5981 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
5982 case Intrinsic::nvvm_tcgen05_mma_sp_shared:
5983 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
5984 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
5985 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale:
5986 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale_block32:
5987 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block16:
5988 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block32:
5989 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale:
5990 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale_block32:
5991 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d:
5992 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
5993 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
5994 case Intrinsic::nvvm_tcgen05_mma_sp_tensor:
5995 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_ashift:
5996 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
5997 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
5998 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
5999 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
6000 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale:
6001 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale_block32:
6002 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block16:
6003 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block32:
6004 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale:
6005 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale_block32:
6006 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d:
6007 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_ashift:
6008 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
6009 case Intrinsic::
6010 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
6011 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
6012 case Intrinsic::
6013 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
6014 case Intrinsic::nvvm_tcgen05_mma_tensor:
6015 case Intrinsic::nvvm_tcgen05_mma_tensor_ashift:
6016 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
6017 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
6018 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
6019 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
6020 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale:
6021 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale_block32:
6022 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block16:
6023 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block32:
6024 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale:
6025 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale_block32:
6026 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d:
6027 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_ashift:
6028 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
6029 case Intrinsic::
6030 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
6031 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
6032 case Intrinsic::
6033 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift: {
6034 SmallVector<Value *, 12> Args(CI->args());
6035 Args.push_back(Elt: Builder.getInt32(C: 0)); // collector_usage_b = discard(0)
6036 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6037 break;
6038 }
6039 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6040 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6041 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6042 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6043 NewCall =
6044 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6045 Builder.getFalse()});
6046 break;
6047 case Intrinsic::riscv_sha256sig0:
6048 case Intrinsic::riscv_sha256sig1:
6049 case Intrinsic::riscv_sha256sum0:
6050 case Intrinsic::riscv_sha256sum1:
6051 case Intrinsic::riscv_sm3p0:
6052 case Intrinsic::riscv_sm3p1: {
6053 // The last argument to these intrinsics used to be i8 and changed to i32.
6054 // The type overload for sm4ks and sm4ed was removed.
6055 if (!CI->getType()->isIntegerTy(BitWidth: 64))
6056 return;
6057
6058 Value *Arg =
6059 Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Builder.getInt32Ty());
6060
6061 NewCall = Builder.CreateCall(Callee: NewFn, Args: Arg);
6062 Value *Res =
6063 Builder.CreateIntCast(V: NewCall, DestTy: CI->getType(), /*isSigned*/ true);
6064 NewCall->takeName(V: CI);
6065 CI->replaceAllUsesWith(V: Res);
6066 CI->eraseFromParent();
6067 return;
6068 }
6069
6070 case Intrinsic::x86_xop_vfrcz_ss:
6071 case Intrinsic::x86_xop_vfrcz_sd:
6072 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 1)});
6073 break;
6074
6075 case Intrinsic::x86_xop_vpermil2pd:
6076 case Intrinsic::x86_xop_vpermil2ps:
6077 case Intrinsic::x86_xop_vpermil2pd_256:
6078 case Intrinsic::x86_xop_vpermil2ps_256: {
6079 SmallVector<Value *, 4> Args(CI->args());
6080 VectorType *FltIdxTy = cast<VectorType>(Val: Args[2]->getType());
6081 VectorType *IntIdxTy = VectorType::getInteger(VTy: FltIdxTy);
6082 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: IntIdxTy);
6083 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6084 break;
6085 }
6086
6087 case Intrinsic::x86_sse41_ptestc:
6088 case Intrinsic::x86_sse41_ptestz:
6089 case Intrinsic::x86_sse41_ptestnzc: {
6090 // The arguments for these intrinsics used to be v4f32, and changed
6091 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6092 // So, the only thing required is a bitcast for both arguments.
6093 // First, check the arguments have the old type.
6094 Value *Arg0 = CI->getArgOperand(i: 0);
6095 if (Arg0->getType() != FixedVectorType::get(ElementType: Type::getFloatTy(C), NumElts: 4))
6096 return;
6097
6098 // Old intrinsic, add bitcasts
6099 Value *Arg1 = CI->getArgOperand(i: 1);
6100
6101 auto *NewVecTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C), NumElts: 2);
6102
6103 Value *BC0 = Builder.CreateBitCast(V: Arg0, DestTy: NewVecTy, Name: "cast");
6104 Value *BC1 = Builder.CreateBitCast(V: Arg1, DestTy: NewVecTy, Name: "cast");
6105
6106 NewCall = Builder.CreateCall(Callee: NewFn, Args: {BC0, BC1});
6107 break;
6108 }
6109
6110 case Intrinsic::x86_rdtscp: {
6111 // This used to take 1 arguments. If we have no arguments, it is already
6112 // upgraded.
6113 if (CI->getNumOperands() == 0)
6114 return;
6115
6116 NewCall = Builder.CreateCall(Callee: NewFn);
6117 // Extract the second result and store it.
6118 Value *Data = Builder.CreateExtractValue(Agg: NewCall, Idxs: 1);
6119 Builder.CreateAlignedStore(Val: Data, Ptr: CI->getArgOperand(i: 0), Align: Align(1));
6120 // Replace the original call result with the first result of the new call.
6121 Value *TSC = Builder.CreateExtractValue(Agg: NewCall, Idxs: 0);
6122
6123 NewCall->takeName(V: CI);
6124 CI->replaceAllUsesWith(V: TSC);
6125 CI->eraseFromParent();
6126 return;
6127 }
6128
6129 case Intrinsic::x86_sse41_insertps:
6130 case Intrinsic::x86_sse41_dppd:
6131 case Intrinsic::x86_sse41_dpps:
6132 case Intrinsic::x86_sse41_mpsadbw:
6133 case Intrinsic::x86_avx_dp_ps_256:
6134 case Intrinsic::x86_avx2_mpsadbw: {
6135 // Need to truncate the last argument from i32 to i8 -- this argument models
6136 // an inherently 8-bit immediate operand to these x86 instructions.
6137 SmallVector<Value *, 4> Args(CI->args());
6138
6139 // Replace the last argument with a trunc.
6140 Args.back() = Builder.CreateTrunc(V: Args.back(), DestTy: Type::getInt8Ty(C), Name: "trunc");
6141 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6142 break;
6143 }
6144
6145 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6146 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6147 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6148 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6149 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6150 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6151 SmallVector<Value *, 4> Args(CI->args());
6152 unsigned NumElts =
6153 cast<FixedVectorType>(Val: Args[0]->getType())->getNumElements();
6154 Args[3] = getX86MaskVec(Builder, Mask: Args[3], NumElts);
6155
6156 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6157 Value *Res = applyX86MaskOn1BitsVec(Builder, Vec: NewCall, Mask: nullptr);
6158
6159 NewCall->takeName(V: CI);
6160 CI->replaceAllUsesWith(V: Res);
6161 CI->eraseFromParent();
6162 return;
6163 }
6164
6165 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6166 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6167 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6168 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6169 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6170 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6171 SmallVector<Value *, 4> Args(CI->args());
6172 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
6173 if (NewFn->getIntrinsicID() ==
6174 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6175 Args[1] = Builder.CreateBitCast(
6176 V: Args[1], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6177
6178 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6179 Value *Res = Builder.CreateBitCast(
6180 V: NewCall, DestTy: FixedVectorType::get(ElementType: Builder.getInt16Ty(), NumElts));
6181
6182 NewCall->takeName(V: CI);
6183 CI->replaceAllUsesWith(V: Res);
6184 CI->eraseFromParent();
6185 return;
6186 }
6187 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6188 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6189 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6190 SmallVector<Value *, 4> Args(CI->args());
6191 unsigned NumElts =
6192 cast<FixedVectorType>(Val: CI->getType())->getNumElements() * 2;
6193 Args[1] = Builder.CreateBitCast(
6194 V: Args[1], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6195 Args[2] = Builder.CreateBitCast(
6196 V: Args[2], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6197
6198 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6199 break;
6200 }
6201
6202 case Intrinsic::thread_pointer: {
6203 NewCall = Builder.CreateCall(Callee: NewFn, Args: {});
6204 break;
6205 }
6206
6207 case Intrinsic::memcpy:
6208 case Intrinsic::memmove:
6209 case Intrinsic::memset: {
6210 // We have to make sure that the call signature is what we're expecting.
6211 // We only want to change the old signatures by removing the alignment arg:
6212 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6213 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6214 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6215 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6216 // Note: i8*'s in the above can be any pointer type
6217 if (CI->arg_size() != 5) {
6218 DefaultCase();
6219 return;
6220 }
6221 // Remove alignment argument (3), and add alignment attributes to the
6222 // dest/src pointers.
6223 Value *Args[4] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6224 CI->getArgOperand(i: 2), CI->getArgOperand(i: 4)};
6225 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6226 AttributeList OldAttrs = CI->getAttributes();
6227 AttributeList NewAttrs = AttributeList::get(
6228 C, FnAttrs: OldAttrs.getFnAttrs(), RetAttrs: OldAttrs.getRetAttrs(),
6229 ArgAttrs: {OldAttrs.getParamAttrs(ArgNo: 0), OldAttrs.getParamAttrs(ArgNo: 1),
6230 OldAttrs.getParamAttrs(ArgNo: 2), OldAttrs.getParamAttrs(ArgNo: 4)});
6231 NewCall->setAttributes(NewAttrs);
6232 auto *MemCI = cast<MemIntrinsic>(Val: NewCall);
6233 // All mem intrinsics support dest alignment.
6234 const ConstantInt *Align = cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
6235 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6236 // Memcpy/Memmove also support source alignment.
6237 if (auto *MTI = dyn_cast<MemTransferInst>(Val: MemCI))
6238 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6239 break;
6240 }
6241
6242 case Intrinsic::masked_load:
6243 case Intrinsic::masked_gather:
6244 case Intrinsic::masked_store:
6245 case Intrinsic::masked_scatter: {
6246 if (CI->arg_size() != 4) {
6247 DefaultCase();
6248 return;
6249 }
6250
6251 auto GetMaybeAlign = [](Value *Op) {
6252 if (auto *CI = dyn_cast<ConstantInt>(Val: Op)) {
6253 uint64_t Val = CI->getZExtValue();
6254 if (Val == 0)
6255 return MaybeAlign();
6256 if (isPowerOf2_64(Value: Val))
6257 return MaybeAlign(Val);
6258 }
6259 reportFatalUsageError(reason: "Invalid alignment argument");
6260 };
6261 auto GetAlign = [&](Value *Op) {
6262 MaybeAlign Align = GetMaybeAlign(Op);
6263 if (Align)
6264 return *Align;
6265 reportFatalUsageError(reason: "Invalid zero alignment argument");
6266 };
6267
6268 const DataLayout &DL = CI->getDataLayout();
6269 switch (NewFn->getIntrinsicID()) {
6270 case Intrinsic::masked_load:
6271 NewCall = Builder.CreateMaskedLoad(
6272 Ty: CI->getType(), Ptr: CI->getArgOperand(i: 0), Alignment: GetAlign(CI->getArgOperand(i: 1)),
6273 Mask: CI->getArgOperand(i: 2), PassThru: CI->getArgOperand(i: 3));
6274 break;
6275 case Intrinsic::masked_gather:
6276 NewCall = Builder.CreateMaskedGather(
6277 Ty: CI->getType(), Ptrs: CI->getArgOperand(i: 0),
6278 Alignment: DL.getValueOrABITypeAlignment(Alignment: GetMaybeAlign(CI->getArgOperand(i: 1)),
6279 Ty: CI->getType()->getScalarType()),
6280 Mask: CI->getArgOperand(i: 2), PassThru: CI->getArgOperand(i: 3));
6281 break;
6282 case Intrinsic::masked_store:
6283 NewCall = Builder.CreateMaskedStore(
6284 Val: CI->getArgOperand(i: 0), Ptr: CI->getArgOperand(i: 1),
6285 Alignment: GetAlign(CI->getArgOperand(i: 2)), Mask: CI->getArgOperand(i: 3));
6286 break;
6287 case Intrinsic::masked_scatter:
6288 NewCall = Builder.CreateMaskedScatter(
6289 Val: CI->getArgOperand(i: 0), Ptrs: CI->getArgOperand(i: 1),
6290 Alignment: DL.getValueOrABITypeAlignment(
6291 Alignment: GetMaybeAlign(CI->getArgOperand(i: 2)),
6292 Ty: CI->getArgOperand(i: 0)->getType()->getScalarType()),
6293 Mask: CI->getArgOperand(i: 3));
6294 break;
6295 default:
6296 llvm_unreachable("Unexpected intrinsic ID");
6297 }
6298 // Previous metadata is still valid.
6299 NewCall->copyMetadata(SrcInst: *CI);
6300 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
6301 break;
6302 }
6303
6304 case Intrinsic::lifetime_start:
6305 case Intrinsic::lifetime_end: {
6306 if (CI->arg_size() != 2) {
6307 DefaultCase();
6308 return;
6309 }
6310
6311 Value *Ptr = CI->getArgOperand(i: 1);
6312 // Try to strip pointer casts, such that the lifetime works on an alloca.
6313 Ptr = Ptr->stripPointerCasts();
6314 if (isa<AllocaInst>(Val: Ptr)) {
6315 // Don't use NewFn, as we might have looked through an addrspacecast.
6316 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6317 NewCall = Builder.CreateLifetimeStart(Ptr);
6318 else
6319 NewCall = Builder.CreateLifetimeEnd(Ptr);
6320 break;
6321 }
6322
6323 // Otherwise remove the lifetime marker.
6324 CI->eraseFromParent();
6325 return;
6326 }
6327
6328 case Intrinsic::x86_avx512_vpdpbusd_128:
6329 case Intrinsic::x86_avx512_vpdpbusd_256:
6330 case Intrinsic::x86_avx512_vpdpbusd_512:
6331 case Intrinsic::x86_avx512_vpdpbusds_128:
6332 case Intrinsic::x86_avx512_vpdpbusds_256:
6333 case Intrinsic::x86_avx512_vpdpbusds_512:
6334 case Intrinsic::x86_avx2_vpdpbssd_128:
6335 case Intrinsic::x86_avx2_vpdpbssd_256:
6336 case Intrinsic::x86_avx10_vpdpbssd_512:
6337 case Intrinsic::x86_avx2_vpdpbssds_128:
6338 case Intrinsic::x86_avx2_vpdpbssds_256:
6339 case Intrinsic::x86_avx10_vpdpbssds_512:
6340 case Intrinsic::x86_avx2_vpdpbsud_128:
6341 case Intrinsic::x86_avx2_vpdpbsud_256:
6342 case Intrinsic::x86_avx10_vpdpbsud_512:
6343 case Intrinsic::x86_avx2_vpdpbsuds_128:
6344 case Intrinsic::x86_avx2_vpdpbsuds_256:
6345 case Intrinsic::x86_avx10_vpdpbsuds_512:
6346 case Intrinsic::x86_avx2_vpdpbuud_128:
6347 case Intrinsic::x86_avx2_vpdpbuud_256:
6348 case Intrinsic::x86_avx10_vpdpbuud_512:
6349 case Intrinsic::x86_avx2_vpdpbuuds_128:
6350 case Intrinsic::x86_avx2_vpdpbuuds_256:
6351 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6352 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6353 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6354 CI->getArgOperand(i: 2)};
6355 Type *NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: NumElts, Scalable: false);
6356 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
6357 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
6358
6359 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6360 break;
6361 }
6362 case Intrinsic::x86_avx512_vpdpwssd_128:
6363 case Intrinsic::x86_avx512_vpdpwssd_256:
6364 case Intrinsic::x86_avx512_vpdpwssd_512:
6365 case Intrinsic::x86_avx512_vpdpwssds_128:
6366 case Intrinsic::x86_avx512_vpdpwssds_256:
6367 case Intrinsic::x86_avx512_vpdpwssds_512:
6368 case Intrinsic::x86_avx2_vpdpwsud_128:
6369 case Intrinsic::x86_avx2_vpdpwsud_256:
6370 case Intrinsic::x86_avx10_vpdpwsud_512:
6371 case Intrinsic::x86_avx2_vpdpwsuds_128:
6372 case Intrinsic::x86_avx2_vpdpwsuds_256:
6373 case Intrinsic::x86_avx10_vpdpwsuds_512:
6374 case Intrinsic::x86_avx2_vpdpwusd_128:
6375 case Intrinsic::x86_avx2_vpdpwusd_256:
6376 case Intrinsic::x86_avx10_vpdpwusd_512:
6377 case Intrinsic::x86_avx2_vpdpwusds_128:
6378 case Intrinsic::x86_avx2_vpdpwusds_256:
6379 case Intrinsic::x86_avx10_vpdpwusds_512:
6380 case Intrinsic::x86_avx2_vpdpwuud_128:
6381 case Intrinsic::x86_avx2_vpdpwuud_256:
6382 case Intrinsic::x86_avx10_vpdpwuud_512:
6383 case Intrinsic::x86_avx2_vpdpwuuds_128:
6384 case Intrinsic::x86_avx2_vpdpwuuds_256:
6385 case Intrinsic::x86_avx10_vpdpwuuds_512:
6386 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6387 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6388 CI->getArgOperand(i: 2)};
6389 Type *NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: NumElts, Scalable: false);
6390 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
6391 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
6392
6393 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6394 break;
6395 }
6396 assert(NewCall && "Should have either set this variable or returned through "
6397 "the default case");
6398 NewCall->takeName(V: CI);
6399 CI->replaceAllUsesWith(V: NewCall);
6400 CI->eraseFromParent();
6401}
6402
6403void llvm::UpgradeCallsToIntrinsic(Function *F) {
6404 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6405
6406 // Check if this function should be upgraded and get the replacement function
6407 // if there is one.
6408 Function *NewFn;
6409 if (UpgradeIntrinsicFunction(F, NewFn)) {
6410 // Replace all users of the old function with the new function or new
6411 // instructions. This is not a range loop because the call is deleted.
6412 for (User *U : make_early_inc_range(Range: F->users()))
6413 if (CallBase *CB = dyn_cast<CallBase>(Val: U))
6414 UpgradeIntrinsicCall(CI: CB, NewFn);
6415
6416 // Remove old function, no longer used, from the module.
6417 if (F != NewFn)
6418 F->eraseFromParent();
6419 }
6420}
6421
6422MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6423 const unsigned NumOperands = MD.getNumOperands();
6424 if (NumOperands == 0)
6425 return &MD; // Invalid, punt to a verifier error.
6426
6427 // Check if the tag uses struct-path aware TBAA format.
6428 if (isa<MDNode>(Val: MD.getOperand(I: 0)) && NumOperands >= 3)
6429 return &MD;
6430
6431 auto &Context = MD.getContext();
6432 if (NumOperands == 3) {
6433 Metadata *Elts[] = {MD.getOperand(I: 0), MD.getOperand(I: 1)};
6434 MDNode *ScalarType = MDNode::get(Context, MDs: Elts);
6435 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6436 Metadata *Elts2[] = {ScalarType, ScalarType,
6437 ConstantAsMetadata::get(
6438 C: Constant::getNullValue(Ty: Type::getInt64Ty(C&: Context))),
6439 MD.getOperand(I: 2)};
6440 return MDNode::get(Context, MDs: Elts2);
6441 }
6442 // Create a MDNode <MD, MD, offset 0>
6443 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(C: Constant::getNullValue(
6444 Ty: Type::getInt64Ty(C&: Context)))};
6445 return MDNode::get(Context, MDs: Elts);
6446}
6447
6448Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6449 Instruction *&Temp) {
6450 if (Opc != Instruction::BitCast)
6451 return nullptr;
6452
6453 Temp = nullptr;
6454 Type *SrcTy = V->getType();
6455 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6456 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6457 LLVMContext &Context = V->getContext();
6458
6459 // We have no information about target data layout, so we assume that
6460 // the maximum pointer size is 64bit.
6461 Type *MidTy = Type::getInt64Ty(C&: Context);
6462 Temp = CastInst::Create(Instruction::PtrToInt, S: V, Ty: MidTy);
6463
6464 return CastInst::Create(Instruction::IntToPtr, S: Temp, Ty: DestTy);
6465 }
6466
6467 return nullptr;
6468}
6469
6470Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6471 if (Opc != Instruction::BitCast)
6472 return nullptr;
6473
6474 Type *SrcTy = C->getType();
6475 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6476 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6477 LLVMContext &Context = C->getContext();
6478
6479 // We have no information about target data layout, so we assume that
6480 // the maximum pointer size is 64bit.
6481 Type *MidTy = Type::getInt64Ty(C&: Context);
6482
6483 return ConstantExpr::getIntToPtr(C: ConstantExpr::getPtrToInt(C, Ty: MidTy),
6484 Ty: DestTy);
6485 }
6486
6487 return nullptr;
6488}
6489
6490static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6491 if (Flag.getNumOperands() < 3)
6492 return std::nullopt;
6493 if (MDString *Name = dyn_cast_or_null<MDString>(Val: Flag.getOperand(I: 1)))
6494 return Name->getString();
6495 return std::nullopt;
6496}
6497
6498/// Check the debug info version number, if it is out-dated, drop the debug
6499/// info. Return true if module is modified.
6500bool llvm::UpgradeDebugInfo(Module &M) {
6501 if (DisableAutoUpgradeDebugInfo)
6502 return false;
6503
6504 llvm::TimeTraceScope timeScope("Upgrade debug info");
6505 // We need to get metadata before the module is verified (i.e., getModuleFlag
6506 // makes assumptions that we haven't verified yet). Carefully extract the flag
6507 // from the metadata.
6508 unsigned Version = 0;
6509 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6510 auto OpIt = find_if(Range: ModFlags->operands(), P: [](const MDNode *Flag) {
6511 if (auto Name = getModuleFlagNameSafely(Flag: *Flag))
6512 return *Name == "Debug Info Version";
6513 return false;
6514 });
6515 if (OpIt != ModFlags->op_end()) {
6516 const MDOperand &ValOp = (*OpIt)->getOperand(I: 2);
6517 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(MD: ValOp))
6518 Version = CI->getZExtValue();
6519 }
6520 }
6521
6522 if (Version == DEBUG_METADATA_VERSION) {
6523 bool BrokenDebugInfo = false;
6524 if (verifyModule(M, OS: &llvm::errs(), BrokenDebugInfo: &BrokenDebugInfo))
6525 report_fatal_error(reason: "Broken module found, compilation aborted!");
6526 if (!BrokenDebugInfo)
6527 // Everything is ok.
6528 return false;
6529 else {
6530 // Diagnose malformed debug info.
6531 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6532 M.getContext().diagnose(DI: Diag);
6533 }
6534 }
6535 bool Modified = StripDebugInfo(M);
6536 if (Modified && Version != DEBUG_METADATA_VERSION) {
6537 // Diagnose a version mismatch.
6538 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6539 M.getContext().diagnose(DI: DiagVersion);
6540 }
6541 return Modified;
6542}
6543
6544static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6545 GlobalValue *GV, const Metadata *V) {
6546 Function *F = cast<Function>(Val: GV);
6547
6548 constexpr StringLiteral DefaultValue = "1";
6549 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6550 unsigned Length = 0;
6551
6552 if (F->hasFnAttribute(Kind: Attr)) {
6553 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6554 // parse these elements placing them into Vect3
6555 StringRef S = F->getFnAttribute(Kind: Attr).getValueAsString();
6556 for (; Length < 3 && !S.empty(); Length++) {
6557 auto [Part, Rest] = S.split(Separator: ',');
6558 Vect3[Length] = Part.trim();
6559 S = Rest;
6560 }
6561 }
6562
6563 const unsigned Dim = DimC - 'x';
6564 assert(Dim < 3 && "Unexpected dim char");
6565
6566 const uint64_t VInt = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6567
6568 // local variable required for StringRef in Vect3 to point to.
6569 const std::string VStr = llvm::utostr(X: VInt);
6570 Vect3[Dim] = VStr;
6571 Length = std::max(a: Length, b: Dim + 1);
6572
6573 const std::string NewAttr = llvm::join(R: ArrayRef(Vect3, Length), Separator: ",");
6574 F->addFnAttr(Kind: Attr, Val: NewAttr);
6575}
6576
6577static inline bool isXYZ(StringRef S) {
6578 return S == "x" || S == "y" || S == "z";
6579}
6580
6581bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6582 const Metadata *V) {
6583 if (K == "kernel") {
6584 if (!mdconst::extract<ConstantInt>(MD&: V)->isZero())
6585 cast<Function>(Val: GV)->setCallingConv(CallingConv::PTX_Kernel);
6586 return true;
6587 }
6588 if (K == "align") {
6589 // V is a bitfeild specifying two 16-bit values. The alignment value is
6590 // specfied in low 16-bits, The index is specified in the high bits. For the
6591 // index, 0 indicates the return value while higher values correspond to
6592 // each parameter (idx = param + 1).
6593 const uint64_t AlignIdxValuePair =
6594 mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6595 const unsigned Idx = (AlignIdxValuePair >> 16);
6596 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6597 cast<Function>(Val: GV)->addAttributeAtIndex(
6598 i: Idx, Attr: Attribute::getWithStackAlignment(Context&: GV->getContext(), Alignment: StackAlign));
6599 return true;
6600 }
6601 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6602 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6603 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: llvm::utostr(X: CV));
6604 return true;
6605 }
6606 if (K == "minctasm") {
6607 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6608 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MinCTASm, Val: llvm::utostr(X: CV));
6609 return true;
6610 }
6611 if (K == "maxnreg") {
6612 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6613 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MaxNReg, Val: llvm::utostr(X: CV));
6614 return true;
6615 }
6616 if (K.consume_front(Prefix: "maxntid") && isXYZ(S: K)) {
6617 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::MaxNTID, DimC: K[0], GV, V);
6618 return true;
6619 }
6620 if (K.consume_front(Prefix: "reqntid") && isXYZ(S: K)) {
6621 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::ReqNTID, DimC: K[0], GV, V);
6622 return true;
6623 }
6624 if (K.consume_front(Prefix: "cluster_dim_") && isXYZ(S: K)) {
6625 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::ClusterDim, DimC: K[0], GV, V);
6626 return true;
6627 }
6628 if (K == "grid_constant") {
6629 const auto Attr = Attribute::get(Context&: GV->getContext(), Kind: NVVMAttr::GridConstant);
6630 for (const auto &Op : cast<MDNode>(Val: V)->operands()) {
6631 // For some reason, the index is 1-based in the metadata. Good thing we're
6632 // able to auto-upgrade it!
6633 const auto Index = mdconst::extract<ConstantInt>(MD: Op)->getZExtValue() - 1;
6634 cast<Function>(Val: GV)->addParamAttr(ArgNo: Index, Attr);
6635 }
6636 return true;
6637 }
6638
6639 return false;
6640}
6641
6642void llvm::UpgradeNVVMAnnotations(Module &M) {
6643 NamedMDNode *NamedMD = M.getNamedMetadata(Name: "nvvm.annotations");
6644 if (!NamedMD)
6645 return;
6646
6647 SmallVector<MDNode *, 8> NewNodes;
6648 SmallPtrSet<const MDNode *, 8> SeenNodes;
6649 for (MDNode *MD : NamedMD->operands()) {
6650 if (!SeenNodes.insert(Ptr: MD).second)
6651 continue;
6652
6653 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD: MD->getOperand(I: 0));
6654 if (!GV)
6655 continue;
6656
6657 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6658
6659 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(I: 0)};
6660 // Each nvvm.annotations metadata entry will be of the following form:
6661 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6662 // start index = 1, to skip the global variable key
6663 // increment = 2, to skip the value for each property-value pairs
6664 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6665 MDString *K = cast<MDString>(Val: MD->getOperand(I: j));
6666 const MDOperand &V = MD->getOperand(I: j + 1);
6667 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K: K->getString(), V);
6668 if (!Upgraded)
6669 NewOperands.append(IL: {K, V});
6670 }
6671
6672 if (NewOperands.size() > 1)
6673 NewNodes.push_back(Elt: MDNode::get(Context&: M.getContext(), MDs: NewOperands));
6674 }
6675
6676 NamedMD->clearOperands();
6677 for (MDNode *N : NewNodes)
6678 NamedMD->addOperand(M: N);
6679}
6680
6681/// This checks for objc retain release marker which should be upgraded. It
6682/// returns true if module is modified.
6683static bool upgradeRetainReleaseMarker(Module &M) {
6684 bool Changed = false;
6685 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6686 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(Name: MarkerKey);
6687 if (ModRetainReleaseMarker) {
6688 MDNode *Op = ModRetainReleaseMarker->getOperand(i: 0);
6689 if (Op) {
6690 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 0));
6691 if (ID) {
6692 SmallVector<StringRef, 4> ValueComp;
6693 ID->getString().split(A&: ValueComp, Separator: "#");
6694 if (ValueComp.size() == 2) {
6695 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6696 ID = MDString::get(Context&: M.getContext(), Str: NewValue);
6697 }
6698 M.addModuleFlag(Behavior: Module::Error, Key: MarkerKey, Val: ID);
6699 M.eraseNamedMetadata(NMD: ModRetainReleaseMarker);
6700 Changed = true;
6701 }
6702 }
6703 }
6704 return Changed;
6705}
6706
6707void llvm::UpgradeARCRuntime(Module &M) {
6708 // This lambda converts normal function calls to ARC runtime functions to
6709 // intrinsic calls.
6710 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6711 llvm::Intrinsic::ID IntrinsicFunc) {
6712 Function *Fn = M.getFunction(Name: OldFunc);
6713
6714 if (!Fn)
6715 return;
6716
6717 Function *NewFn =
6718 llvm::Intrinsic::getOrInsertDeclaration(M: &M, id: IntrinsicFunc);
6719
6720 for (User *U : make_early_inc_range(Range: Fn->users())) {
6721 CallInst *CI = dyn_cast<CallInst>(Val: U);
6722 if (!CI || CI->getCalledFunction() != Fn)
6723 continue;
6724
6725 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6726 FunctionType *NewFuncTy = NewFn->getFunctionType();
6727 SmallVector<Value *, 2> Args;
6728
6729 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6730 // value to the return type of the old function.
6731 if (NewFuncTy->getReturnType() != CI->getType() &&
6732 !CastInst::castIsValid(op: Instruction::BitCast, S: CI,
6733 DstTy: NewFuncTy->getReturnType()))
6734 continue;
6735
6736 bool InvalidCast = false;
6737
6738 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6739 Value *Arg = CI->getArgOperand(i: I);
6740
6741 // Bitcast argument to the parameter type of the new function if it's
6742 // not a variadic argument.
6743 if (I < NewFuncTy->getNumParams()) {
6744 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6745 // to the parameter type of the new function.
6746 if (!CastInst::castIsValid(op: Instruction::BitCast, S: Arg,
6747 DstTy: NewFuncTy->getParamType(i: I))) {
6748 InvalidCast = true;
6749 break;
6750 }
6751 Arg = Builder.CreateBitCast(V: Arg, DestTy: NewFuncTy->getParamType(i: I));
6752 }
6753 Args.push_back(Elt: Arg);
6754 }
6755
6756 if (InvalidCast)
6757 continue;
6758
6759 // Create a call instruction that calls the new function.
6760 CallInst *NewCall = Builder.CreateCall(FTy: NewFuncTy, Callee: NewFn, Args);
6761 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
6762 NewCall->takeName(V: CI);
6763
6764 // Bitcast the return value back to the type of the old call.
6765 Value *NewRetVal = Builder.CreateBitCast(V: NewCall, DestTy: CI->getType());
6766
6767 if (!CI->use_empty())
6768 CI->replaceAllUsesWith(V: NewRetVal);
6769 CI->eraseFromParent();
6770 }
6771
6772 if (Fn->use_empty())
6773 Fn->eraseFromParent();
6774 };
6775
6776 // Unconditionally convert a call to "clang.arc.use" to a call to
6777 // "llvm.objc.clang.arc.use".
6778 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6779
6780 // Upgrade the retain release marker. If there is no need to upgrade
6781 // the marker, that means either the module is already new enough to contain
6782 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6783 if (!upgradeRetainReleaseMarker(M))
6784 return;
6785
6786 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6787 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6788 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6789 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6790 {"objc_autoreleaseReturnValue",
6791 llvm::Intrinsic::objc_autoreleaseReturnValue},
6792 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6793 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6794 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6795 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6796 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6797 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6798 {"objc_release", llvm::Intrinsic::objc_release},
6799 {"objc_retain", llvm::Intrinsic::objc_retain},
6800 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6801 {"objc_retainAutoreleaseReturnValue",
6802 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6803 {"objc_retainAutoreleasedReturnValue",
6804 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
6805 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
6806 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
6807 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
6808 {"objc_unsafeClaimAutoreleasedReturnValue",
6809 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
6810 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
6811 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
6812 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
6813 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
6814 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
6815 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
6816 {"objc_arc_annotation_topdown_bbstart",
6817 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
6818 {"objc_arc_annotation_topdown_bbend",
6819 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
6820 {"objc_arc_annotation_bottomup_bbstart",
6821 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
6822 {"objc_arc_annotation_bottomup_bbend",
6823 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
6824
6825 for (auto &I : RuntimeFuncs)
6826 UpgradeToIntrinsic(I.first, I.second);
6827}
6828
6829// Upgrade the way signing of pointers to init/fini functions is described.
6830//
6831// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
6832// constants, if signing was requested. After the upgrade, these arrays contain
6833// plain function pointers and the desired signing schema is described via a
6834// pair of module flags.
6835//
6836// Note that the upgrade is only performed if all elements of *both* arrays
6837// agree on a common signing schema.
6838static bool upgradePtrauthInitFiniArrays(Module &M) {
6839 // As we cannot always decide whether the particular module should have
6840 // ptrauth-init-fini flags, we have to treat absent flags as having zero
6841 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
6842 // returns as soon as it spots any non-signed init/fini pointer: either we
6843 // should request non-signed pointers (safe to omit both flags) or there is
6844 // no common schema (and thus we do not modify anything).
6845 //
6846 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
6847 // or whether we should request address diversity in addition to the basic
6848 // constant diversity. There is no value representing "decided not to sign"
6849 // for the reasons explained above.
6850 std::optional<bool> UseAddressDisc;
6851
6852 // Do not attempt upgrading if the new module flags already exist.
6853 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6854 for (const MDNode *Flag : ModFlags->operands()) {
6855 std::optional<StringRef> Name = getModuleFlagNameSafely(Flag: *Flag);
6856 if (Name && (*Name == "ptrauth-init-fini" ||
6857 *Name == "ptrauth-init-fini-address-discrimination"))
6858 return false;
6859 }
6860 }
6861
6862 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
6863 constexpr unsigned ExpectedConstDisc = 0xD9D4;
6864 constexpr unsigned ExpectedAddressMarker = 1;
6865
6866 auto *CPA = dyn_cast<ConstantPtrAuth>(Val: CV);
6867 if (!CPA || !CPA->getDiscriminator()->equalsInt(V: ExpectedConstDisc))
6868 return nullptr; // Nothing to upgrade or unknown pattern found.
6869
6870 bool HasAddressDisc;
6871 if (!CPA->hasAddressDiscriminator())
6872 HasAddressDisc = false;
6873 else if (CPA->hasSpecialAddressDiscriminator(Value: ExpectedAddressMarker))
6874 HasAddressDisc = true;
6875 else
6876 return nullptr; // Unknown pattern.
6877
6878 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
6879 return nullptr; // Disagreement with the decided mode.
6880
6881 UseAddressDisc = HasAddressDisc;
6882 return CPA->getPointer();
6883 };
6884
6885 // Do not apply any changes until we know the upgrade is non-ambiguous.
6886 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
6887 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
6888
6889 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
6890 auto *GV = dyn_cast_if_present<GlobalVariable>(Val: M.getNamedValue(Name));
6891 if (!GV || !GV->hasInitializer())
6892 continue; // Skip, but it is okay to upgrade the other variable.
6893
6894 auto *OldStructorsArray = dyn_cast<ConstantArray>(Val: GV->getInitializer());
6895 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
6896 return false;
6897
6898 std::vector<Constant *> NewStructors;
6899 NewStructors.reserve(n: OldStructorsArray->getNumOperands());
6900
6901 for (Use &U : OldStructorsArray->operands()) {
6902 ConstantStruct *Structor = dyn_cast<ConstantStruct>(Val: U.get());
6903 if (!Structor || Structor->getNumOperands() != 3)
6904 return false;
6905
6906 Constant *Prio = Structor->getOperand(i_nocapture: 0);
6907 Constant *Func = Structor->getOperand(i_nocapture: 1);
6908 Constant *Arg = Structor->getOperand(i_nocapture: 2);
6909
6910 Func = UpgradeSinglePointer(Func);
6911 if (!Func)
6912 return false;
6913
6914 NewStructors.push_back(
6915 x: ConstantStruct::get(T: Structor->getType(), V: {Prio, Func, Arg}));
6916 }
6917
6918 Constant *NewInit =
6919 ConstantArray::get(T: OldStructorsArray->getType(), V: NewStructors);
6920 GlobalArraysToUpgrade.emplace_back(Args&: GV, Args&: NewInit);
6921 }
6922
6923 if (GlobalArraysToUpgrade.empty())
6924 return false;
6925 assert(UseAddressDisc.has_value());
6926
6927 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
6928 GV->setInitializer(NewInit);
6929
6930 M.addModuleFlag(Behavior: Module::Error, Key: "ptrauth-init-fini", Val: 1);
6931 M.addModuleFlag(Behavior: Module::Error, Key: "ptrauth-init-fini-address-discrimination",
6932 Val: *UseAddressDisc);
6933
6934 return true;
6935}
6936
6937bool llvm::UpgradeModuleFlags(Module &M) {
6938 bool Changed = false;
6939 Changed |= upgradePtrauthInitFiniArrays(M);
6940
6941 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6942 if (!ModFlags)
6943 return Changed;
6944
6945 bool HasObjCFlag = false, HasClassProperties = false;
6946 bool HasSwiftVersionFlag = false;
6947 uint8_t SwiftMajorVersion, SwiftMinorVersion;
6948 uint32_t SwiftABIVersion;
6949 auto Int8Ty = Type::getInt8Ty(C&: M.getContext());
6950 auto Int32Ty = Type::getInt32Ty(C&: M.getContext());
6951
6952 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
6953 MDNode *Op = ModFlags->getOperand(i: I);
6954 if (Op->getNumOperands() != 3)
6955 continue;
6956 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 1));
6957 if (!ID)
6958 continue;
6959 auto SetBehavior = [&](Module::ModFlagBehavior B) {
6960 Metadata *Ops[3] = {ConstantAsMetadata::get(C: ConstantInt::get(
6961 Ty: Type::getInt32Ty(C&: M.getContext()), V: B)),
6962 MDString::get(Context&: M.getContext(), Str: ID->getString()),
6963 Op->getOperand(I: 2)};
6964 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
6965 Changed = true;
6966 };
6967
6968 if (ID->getString() == "Objective-C Image Info Version")
6969 HasObjCFlag = true;
6970 if (ID->getString() == "Objective-C Class Properties")
6971 HasClassProperties = true;
6972 // Upgrade PIC from Error/Max to Min.
6973 if (ID->getString() == "PIC Level") {
6974 if (auto *Behavior =
6975 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0))) {
6976 uint64_t V = Behavior->getLimitedValue();
6977 if (V == Module::Error || V == Module::Max)
6978 SetBehavior(Module::Min);
6979 }
6980 }
6981 // Upgrade "PIE Level" from Error to Max.
6982 if (ID->getString() == "PIE Level")
6983 if (auto *Behavior =
6984 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0)))
6985 if (Behavior->getLimitedValue() == Module::Error)
6986 SetBehavior(Module::Max);
6987
6988 // Upgrade branch protection and return address signing module flags. The
6989 // module flag behavior for these fields were Error and now they are Min.
6990 if (ID->getString() == "branch-target-enforcement" ||
6991 ID->getString().starts_with(Prefix: "sign-return-address")) {
6992 if (auto *Behavior =
6993 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0))) {
6994 if (Behavior->getLimitedValue() == Module::Error) {
6995 Type *Int32Ty = Type::getInt32Ty(C&: M.getContext());
6996 Metadata *Ops[3] = {
6997 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: Module::Min)),
6998 Op->getOperand(I: 1), Op->getOperand(I: 2)};
6999 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7000 Changed = true;
7001 }
7002 }
7003 }
7004
7005 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7006 // section name so that llvm-lto will not complain about mismatching
7007 // module flags that is functionally the same.
7008 if (ID->getString() == "Objective-C Image Info Section") {
7009 if (auto *Value = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 2))) {
7010 SmallVector<StringRef, 4> ValueComp;
7011 Value->getString().split(A&: ValueComp, Separator: " ");
7012 if (ValueComp.size() != 1) {
7013 std::string NewValue;
7014 for (auto &S : ValueComp)
7015 NewValue += S.str();
7016 Metadata *Ops[3] = {Op->getOperand(I: 0), Op->getOperand(I: 1),
7017 MDString::get(Context&: M.getContext(), Str: NewValue)};
7018 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7019 Changed = true;
7020 }
7021 }
7022 }
7023
7024 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7025 // If the higher bits are set, it adds new module flag for swift info.
7026 if (ID->getString() == "Objective-C Garbage Collection") {
7027 auto Md = dyn_cast<ConstantAsMetadata>(Val: Op->getOperand(I: 2));
7028 if (Md) {
7029 assert(Md->getValue() && "Expected non-empty metadata");
7030 auto Type = Md->getValue()->getType();
7031 if (Type == Int8Ty)
7032 continue;
7033 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7034 if ((Val & 0xff) != Val) {
7035 HasSwiftVersionFlag = true;
7036 SwiftABIVersion = (Val & 0xff00) >> 8;
7037 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7038 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7039 }
7040 Metadata *Ops[3] = {
7041 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty,V: Module::Error)),
7042 Op->getOperand(I: 1),
7043 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int8Ty,V: Val & 0xff))};
7044 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7045 Changed = true;
7046 }
7047 }
7048
7049 if (ID->getString() == "amdgpu_code_object_version") {
7050 Metadata *Ops[3] = {
7051 Op->getOperand(I: 0),
7052 MDString::get(Context&: M.getContext(), Str: "amdhsa_code_object_version"),
7053 Op->getOperand(I: 2)};
7054 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7055 Changed = true;
7056 }
7057
7058 // clang/PowerPC used to use "float-abi" to describe the long double format;
7059 // it has been renamed to "long-double-type", with its values changed to the
7060 // corresponding IR floating-point type names.
7061 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7062 StringRef Format;
7063 if (auto *S = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 2)))
7064 Format = S->getString();
7065
7066 // The "float-abi" key is now reserved for the target-independent
7067 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7068 // (including unrecognized ones, which were never valid) to the default.
7069 if (!FloatABI::parseABIType(S: Format)) {
7070 LongDoubleFormat NewFormat =
7071 StringSwitch<LongDoubleFormat>(Format)
7072 .Case(S: "ieeequad", Value: LongDoubleFormat::IEEEquad)
7073 .Case(S: "ieeedouble", Value: LongDoubleFormat::IEEEdouble)
7074 .Default(Value: LongDoubleFormat::PPCDoubleDouble);
7075 Metadata *Ops[3] = {
7076 Op->getOperand(I: 0),
7077 MDString::get(Context&: M.getContext(), Str: "long-double-type"),
7078 MDString::get(Context&: M.getContext(), Str: getLongDoubleFormatName(Format: NewFormat))};
7079 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7080 Changed = true;
7081 }
7082 }
7083 }
7084
7085 // "Objective-C Class Properties" is recently added for Objective-C. We
7086 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7087 // flag of value 0, so we can correclty downgrade this flag when trying to
7088 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7089 // this module flag.
7090 if (HasObjCFlag && !HasClassProperties) {
7091 M.addModuleFlag(Behavior: llvm::Module::Override, Key: "Objective-C Class Properties",
7092 Val: (uint32_t)0);
7093 Changed = true;
7094 }
7095
7096 if (HasSwiftVersionFlag) {
7097 M.addModuleFlag(Behavior: Module::Error, Key: "Swift ABI Version",
7098 Val: SwiftABIVersion);
7099 M.addModuleFlag(Behavior: Module::Error, Key: "Swift Major Version",
7100 Val: ConstantInt::get(Ty: Int8Ty, V: SwiftMajorVersion));
7101 M.addModuleFlag(Behavior: Module::Error, Key: "Swift Minor Version",
7102 Val: ConstantInt::get(Ty: Int8Ty, V: SwiftMinorVersion));
7103 Changed = true;
7104 }
7105
7106 return Changed;
7107}
7108
7109bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7110 NamedMDNode *CFIConsts = M.getNamedMetadata(Name: "cfi.functions");
7111 // If this metadata has operands, we expect all of them to be either from
7112 // before or from after the format change handled here, so we can bail out
7113 // fast if the first (if any) operands is of the new format.
7114 auto MatchesVersion = [](const MDNode *Op) {
7115 return Op->getNumOperands() >= 3 &&
7116 isa<ConstantAsMetadata>(Val: Op->getOperand(I: 2)) &&
7117 cast<ConstantAsMetadata>(Val: Op->getOperand(I: 2))
7118 ->getType()
7119 ->isIntegerTy(BitWidth: 64);
7120 };
7121
7122 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7123 MatchesVersion(CFIConsts->getOperand(i: 0)))
7124 return false;
7125
7126 bool Changed = false;
7127 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7128 MDNode *Op = CFIConsts->getOperand(i: I);
7129 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7130 assert(Op->getNumOperands() >= 2 &&
7131 "Expected at least 2 operands - name and linkage type");
7132 MDString *NameMD = dyn_cast<MDString>(Val: Op->getOperand(I: 0));
7133 StringRef Name = NameMD->getString();
7134 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7135 GlobalName: GlobalValue::dropLLVMManglingEscape(Name));
7136
7137 SmallVector<Metadata *, 4> Elts;
7138 Elts.push_back(Elt: Op->getOperand(I: 0));
7139 Elts.push_back(Elt: Op->getOperand(I: 1));
7140 Elts.push_back(Elt: ConstantAsMetadata::get(
7141 C: ConstantInt::get(Ty: Type::getInt64Ty(C&: M.getContext()), V: GUID)));
7142
7143 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7144 Elts.push_back(Elt: Op->getOperand(I: J));
7145
7146 CFIConsts->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Elts));
7147 Changed = true;
7148 }
7149
7150 return Changed;
7151}
7152
7153void llvm::UpgradeSectionAttributes(Module &M) {
7154 auto TrimSpaces = [](StringRef Section) -> std::string {
7155 SmallVector<StringRef, 5> Components;
7156 Section.split(A&: Components, Separator: ',');
7157
7158 SmallString<32> Buffer;
7159 raw_svector_ostream OS(Buffer);
7160
7161 for (auto Component : Components)
7162 OS << ',' << Component.trim();
7163
7164 return std::string(OS.str().substr(Start: 1));
7165 };
7166
7167 for (auto &GV : M.globals()) {
7168 if (!GV.hasSection())
7169 continue;
7170
7171 StringRef Section = GV.getSection();
7172
7173 if (!Section.starts_with(Prefix: "__DATA, __objc_catlist"))
7174 continue;
7175
7176 // __DATA, __objc_catlist, regular, no_dead_strip
7177 // __DATA,__objc_catlist,regular,no_dead_strip
7178 GV.setSection(TrimSpaces(Section));
7179 }
7180}
7181
7182namespace {
7183// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7184// callsites within a function that did not also have the strictfp attribute.
7185// Since 10.0, if strict FP semantics are needed within a function, the
7186// function must have the strictfp attribute and all calls within the function
7187// must also have the strictfp attribute. This latter restriction is
7188// necessary to prevent unwanted libcall simplification when a function is
7189// being cloned (such as for inlining).
7190//
7191// The "dangling" strictfp attribute usage was only used to prevent constant
7192// folding and other libcall simplification. The nobuiltin attribute on the
7193// callsite has the same effect.
7194struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7195 StrictFPUpgradeVisitor() = default;
7196
7197 void visitCallBase(CallBase &Call) {
7198 if (!Call.isStrictFP())
7199 return;
7200 if (isa<ConstrainedFPIntrinsic>(Val: &Call))
7201 return;
7202 // If we get here, the caller doesn't have the strictfp attribute
7203 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7204 Call.removeFnAttr(Kind: Attribute::StrictFP);
7205 Call.addFnAttr(Kind: Attribute::NoBuiltin);
7206 }
7207};
7208
7209/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7210struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7211 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7212 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7213
7214 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7215 if (!RMW.isFloatingPointOperation())
7216 return;
7217
7218 MDNode *Empty = MDNode::get(Context&: RMW.getContext(), MDs: {});
7219 RMW.setMetadata(Kind: "amdgpu.no.fine.grained.host.memory", Node: Empty);
7220 RMW.setMetadata(Kind: "amdgpu.no.remote.memory.access", Node: Empty);
7221 RMW.setMetadata(Kind: "amdgpu.ignore.denormal.mode", Node: Empty);
7222 }
7223};
7224} // namespace
7225
7226void llvm::UpgradeFunctionAttributes(Function &F) {
7227 // If a function definition doesn't have the strictfp attribute,
7228 // convert any callsite strictfp attributes to nobuiltin.
7229 if (!F.isDeclaration() && !F.hasFnAttribute(Kind: Attribute::StrictFP)) {
7230 StrictFPUpgradeVisitor SFPV;
7231 SFPV.visit(F);
7232 }
7233
7234 // Remove all incompatibile attributes from function.
7235 F.removeRetAttrs(Attrs: AttributeFuncs::typeIncompatible(
7236 Ty: F.getReturnType(), AS: F.getAttributes().getRetAttrs()));
7237 for (auto &Arg : F.args())
7238 Arg.removeAttrs(
7239 AM: AttributeFuncs::typeIncompatible(Ty: Arg.getType(), AS: Arg.getAttributes()));
7240
7241 bool AddingAttrs = false, RemovingAttrs = false;
7242 AttrBuilder AttrsToAdd(F.getContext());
7243 AttributeMask AttrsToRemove;
7244
7245 // Older versions of LLVM treated an "implicit-section-name" attribute
7246 // similarly to directly setting the section on a Function.
7247 if (Attribute A = F.getFnAttribute(Kind: "implicit-section-name");
7248 A.isValid() && A.isStringAttribute()) {
7249 F.setSection(A.getValueAsString());
7250 AttrsToRemove.addAttribute(A: "implicit-section-name");
7251 RemovingAttrs = true;
7252 }
7253
7254 if (Attribute A = F.getFnAttribute(Kind: "nooutline");
7255 A.isValid() && A.isStringAttribute()) {
7256 AttrsToRemove.addAttribute(A: "nooutline");
7257 AttrsToAdd.addAttribute(Val: Attribute::NoOutline);
7258 AddingAttrs = RemovingAttrs = true;
7259 }
7260
7261 if (Attribute A = F.getFnAttribute(Kind: "uniform-work-group-size");
7262 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7263 AttrsToRemove.addAttribute(A: "uniform-work-group-size");
7264 RemovingAttrs = true;
7265 if (A.getValueAsString() == "true") {
7266 AttrsToAdd.addAttribute(A: "uniform-work-group-size");
7267 AddingAttrs = true;
7268 }
7269 }
7270
7271 if (!F.empty()) {
7272 // For some reason this is called twice, and the first time is before any
7273 // instructions are loaded into the body.
7274
7275 if (Attribute A = F.getFnAttribute(Kind: "amdgpu-unsafe-fp-atomics");
7276 A.isValid()) {
7277
7278 if (A.getValueAsBool()) {
7279 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7280 Visitor.visit(F);
7281 }
7282
7283 // We will leave behind dead attribute uses on external declarations, but
7284 // clang never added these to declarations anyway.
7285 AttrsToRemove.addAttribute(A: "amdgpu-unsafe-fp-atomics");
7286 RemovingAttrs = true;
7287 }
7288 }
7289
7290 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7291 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7292
7293 bool HandleDenormalMode = false;
7294
7295 if (Attribute Attr = F.getFnAttribute(Kind: "denormal-fp-math"); Attr.isValid()) {
7296 DenormalMode ParsedMode = parseDenormalFPAttribute(Str: Attr.getValueAsString());
7297 if (ParsedMode.isValid()) {
7298 DenormalFPMath = ParsedMode;
7299 AttrsToRemove.addAttribute(A: "denormal-fp-math");
7300 AddingAttrs = RemovingAttrs = true;
7301 HandleDenormalMode = true;
7302 }
7303 }
7304
7305 if (Attribute Attr = F.getFnAttribute(Kind: "denormal-fp-math-f32");
7306 Attr.isValid()) {
7307 DenormalMode ParsedMode = parseDenormalFPAttribute(Str: Attr.getValueAsString());
7308 if (ParsedMode.isValid()) {
7309 DenormalFPMathF32 = ParsedMode;
7310 AttrsToRemove.addAttribute(A: "denormal-fp-math-f32");
7311 AddingAttrs = RemovingAttrs = true;
7312 HandleDenormalMode = true;
7313 }
7314 }
7315
7316 if (HandleDenormalMode)
7317 AttrsToAdd.addDenormalFPEnvAttr(
7318 Mode: DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7319
7320 if (RemovingAttrs)
7321 F.removeFnAttrs(Attrs: AttrsToRemove);
7322
7323 if (AddingAttrs)
7324 F.addFnAttrs(Attrs: AttrsToAdd);
7325}
7326
7327// Check if the function attribute is not present and set it.
7328static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7329 StringRef Value) {
7330 if (!F.hasFnAttribute(Kind: FnAttrName))
7331 F.addFnAttr(Kind: FnAttrName, Val: Value);
7332}
7333
7334// Check if the function attribute is not present and set it if needed.
7335// If the attribute is "false" then removes it.
7336// If the attribute is "true" resets it to a valueless attribute.
7337static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7338 if (!F.hasFnAttribute(Kind: FnAttrName)) {
7339 if (Set)
7340 F.addFnAttr(Kind: FnAttrName);
7341 } else {
7342 auto A = F.getFnAttribute(Kind: FnAttrName);
7343 if ("false" == A.getValueAsString())
7344 F.removeFnAttr(Kind: FnAttrName);
7345 else if ("true" == A.getValueAsString()) {
7346 F.removeFnAttr(Kind: FnAttrName);
7347 F.addFnAttr(Kind: FnAttrName);
7348 }
7349 }
7350}
7351
7352void llvm::copyModuleAttrToFunctions(Module &M) {
7353 Triple T(M.getTargetTriple());
7354 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7355 return;
7356
7357 uint64_t BTEValue = 0;
7358 uint64_t BPPLRValue = 0;
7359 uint64_t GCSValue = 0;
7360 uint64_t SRAValue = 0;
7361 uint64_t SRAALLValue = 0;
7362 uint64_t SRABKeyValue = 0;
7363
7364 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7365 if (ModFlags) {
7366 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7367 MDNode *Op = ModFlags->getOperand(i: I);
7368 if (Op->getNumOperands() != 3)
7369 continue;
7370
7371 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 1));
7372 auto *CI = mdconst::dyn_extract<ConstantInt>(MD: Op->getOperand(I: 2));
7373 if (!ID || !CI)
7374 continue;
7375
7376 StringRef IDStr = ID->getString();
7377 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7378 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7379 : IDStr == "guarded-control-stack" ? &GCSValue
7380 : IDStr == "sign-return-address" ? &SRAValue
7381 : IDStr == "sign-return-address-all" ? &SRAALLValue
7382 : IDStr == "sign-return-address-with-bkey"
7383 ? &SRABKeyValue
7384 : nullptr;
7385 if (!ValPtr)
7386 continue;
7387
7388 *ValPtr = CI->getZExtValue();
7389 if (*ValPtr == 2)
7390 return;
7391 }
7392 }
7393
7394 bool BTE = BTEValue == 1;
7395 bool BPPLR = BPPLRValue == 1;
7396 bool GCS = GCSValue == 1;
7397 bool SRA = SRAValue == 1;
7398
7399 StringRef SignTypeValue = "non-leaf";
7400 if (SRA && SRAALLValue == 1)
7401 SignTypeValue = "all";
7402
7403 StringRef SignKeyValue = "a_key";
7404 if (SRA && SRABKeyValue == 1)
7405 SignKeyValue = "b_key";
7406
7407 for (Function &F : M.getFunctionList()) {
7408 if (F.isDeclaration())
7409 continue;
7410
7411 if (SRA) {
7412 setFunctionAttrIfNotSet(F, FnAttrName: "sign-return-address", Value: SignTypeValue);
7413 setFunctionAttrIfNotSet(F, FnAttrName: "sign-return-address-key", Value: SignKeyValue);
7414 } else {
7415 if (auto A = F.getFnAttribute(Kind: "sign-return-address");
7416 A.isValid() && "none" == A.getValueAsString()) {
7417 F.removeFnAttr(Kind: "sign-return-address");
7418 F.removeFnAttr(Kind: "sign-return-address-key");
7419 }
7420 }
7421 ConvertFunctionAttr(F, Set: BTE, FnAttrName: "branch-target-enforcement");
7422 ConvertFunctionAttr(F, Set: BPPLR, FnAttrName: "branch-protection-pauth-lr");
7423 ConvertFunctionAttr(F, Set: GCS, FnAttrName: "guarded-control-stack");
7424 }
7425
7426 if (BTE)
7427 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "branch-target-enforcement", Val: 2);
7428 if (BPPLR)
7429 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "branch-protection-pauth-lr", Val: 2);
7430 if (GCS)
7431 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "guarded-control-stack", Val: 2);
7432 if (SRA) {
7433 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address", Val: 2);
7434 if (SRAALLValue == 1)
7435 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address-all", Val: 2);
7436 if (SRABKeyValue == 1)
7437 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address-with-bkey", Val: 2);
7438 }
7439}
7440
7441/// Return the replacement tags if \p T still uses a removed two-operand form.
7442static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7443 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(MD: T->getOperand(I: 1)))
7444 return nullptr;
7445 auto *Tag = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7446 return Tag ? findBooleanLoopTags(Name: Tag->getString()) : nullptr;
7447}
7448
7449/// Build the single-operand node that replaces a boolean operand: nonzero
7450/// selects the enable tag, zero the disable tag.
7451static Metadata *makeBooleanLoopNode(LLVMContext &C,
7452 const BooleanLoopTags &Tags,
7453 const MDOperand &Op) {
7454 bool Enable = !mdconst::extract<ConstantInt>(MD: Op)->isZero();
7455 return MDTuple::get(Context&: C,
7456 MDs: {MDString::get(Context&: C, Str: Enable ? Tags.Enable : Tags.Disable)});
7457}
7458
7459static bool isOldLoopArgument(Metadata *MD) {
7460 auto *T = dyn_cast_or_null<MDTuple>(Val: MD);
7461 if (!T)
7462 return false;
7463 if (T->getNumOperands() < 1)
7464 return false;
7465 auto *S = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7466 if (!S)
7467 return false;
7468 if (S->getString().starts_with(Prefix: "llvm.vectorizer."))
7469 return true;
7470 return getOldBooleanLoopTags(T) != nullptr;
7471}
7472
7473static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7474 StringRef OldPrefix = "llvm.vectorizer.";
7475 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7476
7477 if (OldTag == "llvm.vectorizer.unroll")
7478 return MDString::get(Context&: C, Str: "llvm.loop.interleave.count");
7479
7480 return MDString::get(
7481 Context&: C, Str: (Twine("llvm.loop.vectorize.") + OldTag.drop_front(N: OldPrefix.size()))
7482 .str());
7483}
7484
7485static Metadata *upgradeLoopArgument(Metadata *MD) {
7486 auto *T = dyn_cast_or_null<MDTuple>(Val: MD);
7487 if (!T)
7488 return MD;
7489 if (T->getNumOperands() < 1)
7490 return MD;
7491 auto *OldTag = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7492 if (!OldTag)
7493 return MD;
7494
7495 LLVMContext &C = T->getContext();
7496
7497 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7498 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7499 return makeBooleanLoopNode(C, Tags: *Tags, Op: T->getOperand(I: 1));
7500
7501 if (!OldTag->getString().starts_with(Prefix: "llvm.vectorizer."))
7502 return MD;
7503
7504 // This has an old tag. Upgrade it.
7505 MDString *NewTag = upgradeLoopTag(C, OldTag: OldTag->getString());
7506
7507 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7508 // vectorize.enable/disable pair, not a two-operand enable node.
7509 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(MD: T->getOperand(I: 1)))
7510 if (const BooleanLoopTags *Tags = findBooleanLoopTags(Name: NewTag->getString()))
7511 return makeBooleanLoopNode(C, Tags: *Tags, Op: T->getOperand(I: 1));
7512
7513 SmallVector<Metadata *, 8> Ops;
7514 Ops.reserve(N: T->getNumOperands());
7515 Ops.push_back(Elt: NewTag);
7516 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7517 Ops.push_back(Elt: T->getOperand(I));
7518
7519 return MDTuple::get(Context&: C, MDs: Ops);
7520}
7521
7522MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7523 auto *T = dyn_cast<MDTuple>(Val: &N);
7524 if (!T)
7525 return &N;
7526
7527 if (none_of(Range: T->operands(), P: isOldLoopArgument))
7528 return &N;
7529
7530 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7531 // any MDNode carrying those tags with more than one operand, so a leftover
7532 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7533 // In-place mutation is safe on distinct MDNodes.
7534 if (T->isDistinct()) {
7535 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7536 auto *OpT = dyn_cast_or_null<MDTuple>(Val: T->getOperand(I));
7537 if (OpT && getOldBooleanLoopTags(T: OpT))
7538 T->replaceOperandWith(I, New: upgradeLoopArgument(MD: OpT));
7539 }
7540 if (none_of(Range: T->operands(), P: isOldLoopArgument))
7541 return &N;
7542 }
7543
7544 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7545 // attachment; the original distinct loop-ID is kept as the first operand.
7546 SmallVector<Metadata *, 8> Ops;
7547 Ops.reserve(N: T->getNumOperands());
7548 for (Metadata *MD : T->operands())
7549 Ops.push_back(Elt: upgradeLoopArgument(MD));
7550
7551 return MDTuple::get(Context&: T->getContext(), MDs: Ops);
7552}
7553
7554std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7555 Triple T(TT);
7556 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7557 // the address space of globals to 1. This does not apply to SPIRV Logical.
7558 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7559 !DL.contains(Other: "-G") && !DL.starts_with(Prefix: "G")) {
7560 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7561 }
7562
7563 if (T.isLoongArch64() || T.isRISCV64()) {
7564 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7565 auto I = DL.find(Str: "-n64-");
7566 if (I != StringRef::npos)
7567 return (DL.take_front(N: I) + "-n32:64-" + DL.drop_front(N: I + 5)).str();
7568 return DL.str();
7569 }
7570
7571 // AMDGPU data layout upgrades.
7572 std::string Res = DL.str();
7573 if (T.isAMDGPU()) {
7574 // Define address spaces for constants.
7575 if (!DL.contains(Other: "-G") && !DL.starts_with(Prefix: "G"))
7576 Res.append(s: Res.empty() ? "G1" : "-G1");
7577
7578 // AMDGCN data layout upgrades.
7579 if (T.isAMDGCN()) {
7580
7581 // Add missing non-integral declarations.
7582 // This goes before adding new address spaces to prevent incoherent string
7583 // values.
7584 if (!DL.contains(Other: "-ni") && !DL.starts_with(Prefix: "ni"))
7585 Res.append(s: "-ni:7:8:9");
7586 // Update ni:7 to ni:7:8:9.
7587 if (DL.ends_with(Suffix: "ni:7"))
7588 Res.append(s: ":8:9");
7589 if (DL.ends_with(Suffix: "ni:7:8"))
7590 Res.append(s: ":9");
7591
7592 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7593 // resources) An empty data layout has already been upgraded to G1 by now.
7594 if (!DL.contains(Other: "-p7") && !DL.starts_with(Prefix: "p7"))
7595 Res.append(s: "-p7:160:256:256:32");
7596 if (!DL.contains(Other: "-p8") && !DL.starts_with(Prefix: "p8"))
7597 Res.append(s: "-p8:128:128:128:48");
7598 constexpr StringRef OldP8("-p8:128:128-");
7599 if (DL.contains(Other: OldP8))
7600 Res.replace(pos: Res.find(svt: OldP8), n1: OldP8.size(), s: "-p8:128:128:128:48-");
7601 if (!DL.contains(Other: "-p9") && !DL.starts_with(Prefix: "p9"))
7602 Res.append(s: "-p9:192:256:256:32");
7603 }
7604
7605 // Upgrade the ELF mangling mode.
7606 if (!DL.contains(Other: "m:e"))
7607 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7608
7609 return Res;
7610 }
7611
7612 if (T.isSystemZ() && !DL.empty()) {
7613 // Make sure the stack alignment is present.
7614 if (!DL.contains(Other: "-S64"))
7615 return "E-S64" + DL.drop_front(N: 1).str();
7616 return DL.str();
7617 }
7618
7619 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7620 // If the datalayout matches the expected format, add pointer size address
7621 // spaces to the datalayout.
7622 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7623 if (!DL.contains(Other: AddrSpaces)) {
7624 SmallVector<StringRef, 4> Groups;
7625 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7626 if (R.match(String: Res, Matches: &Groups))
7627 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7628 }
7629 };
7630
7631 // AArch64 data layout upgrades.
7632 if (T.isAArch64()) {
7633 // Add "-Fn32"
7634 if (!DL.empty() && !DL.contains(Other: "-Fn32"))
7635 Res.append(s: "-Fn32");
7636 AddPtr32Ptr64AddrSpaces();
7637 return Res;
7638 }
7639
7640 if (T.isSPARC() || (T.isMIPS64() && !DL.contains(Other: "m:m")) || T.isPPC64() ||
7641 T.isWasm()) {
7642 // Mips64 with o32 ABI did not add "-i128:128".
7643 // Add "-i128:128"
7644 std::string I64 = "-i64:64";
7645 std::string I128 = "-i128:128";
7646 if (!StringRef(Res).contains(Other: I128)) {
7647 size_t Pos = Res.find(str: I64);
7648 if (Pos != size_t(-1))
7649 Res.insert(pos1: Pos + I64.size(), str: I128);
7650 }
7651 }
7652
7653 if (T.isPPC() && T.isOSAIX() && !DL.contains(Other: "f64:32:64") && !DL.empty()) {
7654 size_t Pos = Res.find(s: "-S128");
7655 if (Pos == StringRef::npos)
7656 Pos = Res.size();
7657 Res.insert(pos: Pos, s: "-f64:32:64");
7658 }
7659
7660 if (!T.isX86())
7661 return Res;
7662
7663 AddPtr32Ptr64AddrSpaces();
7664
7665 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7666 // for i128 operations prior to this being reflected in the data layout, and
7667 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7668 // boundaries, so although this is a breaking change, the upgrade is expected
7669 // to fix more IR than it breaks.
7670 // Intel MCU is an exception and uses 4-byte-alignment.
7671 if (!T.isOSIAMCU()) {
7672 std::string I128 = "-i128:128";
7673 if (StringRef Ref = Res; !Ref.contains(Other: I128)) {
7674 SmallVector<StringRef, 4> Groups;
7675 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7676 if (R.match(String: Res, Matches: &Groups))
7677 Res = (Groups[1] + I128 + Groups[3]).str();
7678 }
7679 }
7680
7681 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7682 // Raising the alignment is safe because Clang did not produce f80 values in
7683 // the MSVC environment before this upgrade was added.
7684 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7685 StringRef Ref = Res;
7686 auto I = Ref.find(Str: "-f80:32-");
7687 if (I != StringRef::npos)
7688 Res = (Ref.take_front(N: I) + "-f80:128-" + Ref.drop_front(N: I + 8)).str();
7689 }
7690
7691 return Res;
7692}
7693
7694void llvm::UpgradeAttributes(AttrBuilder &B) {
7695 StringRef FramePointer;
7696 Attribute A = B.getAttribute(Kind: "no-frame-pointer-elim");
7697 if (A.isValid()) {
7698 // The value can be "true" or "false".
7699 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7700 B.removeAttribute(A: "no-frame-pointer-elim");
7701 }
7702 if (B.contains(A: "no-frame-pointer-elim-non-leaf")) {
7703 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7704 if (FramePointer != "all")
7705 FramePointer = "non-leaf";
7706 B.removeAttribute(A: "no-frame-pointer-elim-non-leaf");
7707 }
7708 if (!FramePointer.empty())
7709 B.addAttribute(A: "frame-pointer", V: FramePointer);
7710
7711 A = B.getAttribute(Kind: "null-pointer-is-valid");
7712 if (A.isValid()) {
7713 // The value can be "true" or "false".
7714 bool NullPointerIsValid = A.getValueAsString() == "true";
7715 B.removeAttribute(A: "null-pointer-is-valid");
7716 if (NullPointerIsValid)
7717 B.addAttribute(Val: Attribute::NullPointerIsValid);
7718 }
7719
7720 A = B.getAttribute(Kind: "uniform-work-group-size");
7721 if (A.isValid()) {
7722 StringRef Val = A.getValueAsString();
7723 if (!Val.empty()) {
7724 bool IsTrue = Val == "true";
7725 B.removeAttribute(A: "uniform-work-group-size");
7726 if (IsTrue)
7727 B.addAttribute(A: "uniform-work-group-size");
7728 }
7729 }
7730}
7731
7732void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7733 // clang.arc.attachedcall bundles are now required to have an operand.
7734 // If they don't, it's okay to drop them entirely: when there is an operand,
7735 // the "attachedcall" is meaningful and required, but without an operand,
7736 // it's just a marker NOP. Dropping it merely prevents an optimization.
7737 erase_if(C&: Bundles, P: [&](OperandBundleDef &OBD) {
7738 return OBD.getTag() == "clang.arc.attachedcall" &&
7739 OBD.inputs().empty();
7740 });
7741}
7742